1 """HTTP/1.1 client library

     2 

     3 <intro stuff goes here>

     4 <other stuff, too>

     5 

     6 HTTPConnection goes through a number of "states", which define when a client

     7 may legally make another request or fetch the response for a particular

     8 request. This diagram details these state transitions:

     9 

    10     (null)

    11       |

    12       | HTTPConnection()

    13       v

    14     Idle

    15       |

    16       | putrequest()

    17       v

    18     Request-started

    19       |

    20       | ( putheader() )*  endheaders()

    21       v

    22     Request-sent

    23       |

    24       | response = getresponse()

    25       v

    26     Unread-response   [Response-headers-read]

    27       |\____________________

    28       |                     |

    29       | response.read()     | putrequest()

    30       v                     v

    31     Idle                  Req-started-unread-response

    32                      ______/|

    33                    /        |

    34    response.read() |        | ( putheader() )*  endheaders()

    35                    v        v

    36        Request-started    Req-sent-unread-response

    37                             |

    38                             | response.read()

    39                             v

    40                           Request-sent

    41 

    42 This diagram presents the following rules:

    43   -- a second request may not be started until {response-headers-read}

    44   -- a response [object] cannot be retrieved until {request-sent}

    45   -- there is no differentiation between an unread response body and a

    46      partially read response body

    47 

    48 Note: this enforcement is applied by the HTTPConnection class. The

    49       HTTPResponse class does not enforce this state machine, which

    50       implies sophisticated clients may accelerate the request/response

    51       pipeline. Caution should be taken, though: accelerating the states

    52       beyond the above pattern may imply knowledge of the server's

    53       connection-close behavior for certain requests. For example, it

    54       is impossible to tell whether the server will close the connection

    55       UNTIL the response headers have been read; this means that further

    56       requests cannot be placed into the pipeline until it is known that

    57       the server will NOT be closing the connection.

    58 

    59 Logical State                  __state            __response

    60 -------------                  -------            ----------

    61 Idle                           _CS_IDLE           None

    62 Request-started                _CS_REQ_STARTED    None

    63 Request-sent                   _CS_REQ_SENT       None

    64 Unread-response                _CS_IDLE           <response_class>

    65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>

    66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>

    67 """

    68 

    69 import errno

    70 import mimetools

    71 import socket

    72 from urlparse import urlsplit

    73 

    74 try:

    75     from cStringIO import StringIO

    76 except ImportError:

    77     from StringIO import StringIO

    78 

    79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",

    80            "HTTPException", "NotConnected", "UnknownProtocol",

    81            "UnknownTransferEncoding", "UnimplementedFileMode",

    82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",

    83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",

    84            "BadStatusLine", "error", "responses"]

    85 

    86 HTTP_PORT = 80

    87 HTTPS_PORT = 443

    88 

    89 _UNKNOWN = 'UNKNOWN'

    90 

    91 # connection states

    92 _CS_IDLE = 'Idle'

    93 _CS_REQ_STARTED = 'Request-started'

    94 _CS_REQ_SENT = 'Request-sent'

    95 

    96 # status codes

    97 # informational

    98 CONTINUE = 100

    99 SWITCHING_PROTOCOLS = 101

   100 PROCESSING = 102

   101 

   102 # successful

   103 OK = 200

   104 CREATED = 201

   105 ACCEPTED = 202

   106 NON_AUTHORITATIVE_INFORMATION = 203

   107 NO_CONTENT = 204

   108 RESET_CONTENT = 205

   109 PARTIAL_CONTENT = 206

   110 MULTI_STATUS = 207

   111 IM_USED = 226

   112 

   113 # redirection

   114 MULTIPLE_CHOICES = 300

   115 MOVED_PERMANENTLY = 301

   116 FOUND = 302

   117 SEE_OTHER = 303

   118 NOT_MODIFIED = 304

   119 USE_PROXY = 305

   120 TEMPORARY_REDIRECT = 307

   121 

   122 # client error

   123 BAD_REQUEST = 400

   124 UNAUTHORIZED = 401

   125 PAYMENT_REQUIRED = 402

   126 FORBIDDEN = 403

   127 NOT_FOUND = 404

   128 METHOD_NOT_ALLOWED = 405

   129 NOT_ACCEPTABLE = 406

   130 PROXY_AUTHENTICATION_REQUIRED = 407

   131 REQUEST_TIMEOUT = 408

   132 CONFLICT = 409

   133 GONE = 410

   134 LENGTH_REQUIRED = 411

   135 PRECONDITION_FAILED = 412

   136 REQUEST_ENTITY_TOO_LARGE = 413

   137 REQUEST_URI_TOO_LONG = 414

   138 UNSUPPORTED_MEDIA_TYPE = 415

   139 REQUESTED_RANGE_NOT_SATISFIABLE = 416

   140 EXPECTATION_FAILED = 417

   141 UNPROCESSABLE_ENTITY = 422

   142 LOCKED = 423

   143 FAILED_DEPENDENCY = 424

   144 UPGRADE_REQUIRED = 426

   145 

   146 # server error

   147 INTERNAL_SERVER_ERROR = 500

   148 NOT_IMPLEMENTED = 501

   149 BAD_GATEWAY = 502

   150 SERVICE_UNAVAILABLE = 503

   151 GATEWAY_TIMEOUT = 504

   152 HTTP_VERSION_NOT_SUPPORTED = 505

   153 INSUFFICIENT_STORAGE = 507

   154 NOT_EXTENDED = 510

   155 

   156 # Mapping status codes to official W3C names

   157 responses = {

   158     100: 'Continue',

   159     101: 'Switching Protocols',

   160 

   161     200: 'OK',

   162     201: 'Created',

   163     202: 'Accepted',

   164     203: 'Non-Authoritative Information',

   165     204: 'No Content',

   166     205: 'Reset Content',

   167     206: 'Partial Content',

   168 

   169     300: 'Multiple Choices',

   170     301: 'Moved Permanently',

   171     302: 'Found',

   172     303: 'See Other',

   173     304: 'Not Modified',

   174     305: 'Use Proxy',

   175     306: '(Unused)',

   176     307: 'Temporary Redirect',

   177 

   178     400: 'Bad Request',

   179     401: 'Unauthorized',

   180     402: 'Payment Required',

   181     403: 'Forbidden',

   182     404: 'Not Found',

   183     405: 'Method Not Allowed',

   184     406: 'Not Acceptable',

   185     407: 'Proxy Authentication Required',

   186     408: 'Request Timeout',

   187     409: 'Conflict',

   188     410: 'Gone',

   189     411: 'Length Required',

   190     412: 'Precondition Failed',

   191     413: 'Request Entity Too Large',

   192     414: 'Request-URI Too Long',

   193     415: 'Unsupported Media Type',

   194     416: 'Requested Range Not Satisfiable',

   195     417: 'Expectation Failed',

   196 

   197     500: 'Internal Server Error',

   198     501: 'Not Implemented',

   199     502: 'Bad Gateway',

   200     503: 'Service Unavailable',

   201     504: 'Gateway Timeout',

   202     505: 'HTTP Version Not Supported',

   203 }

   204 

   205 # maximal amount of data to read at one time in _safe_read

   206 MAXAMOUNT = 1048576

   207 

   208 class HTTPMessage(mimetools.Message):

   209 

   210     def addheader(self, key, value):

   211         """Add header for field key handling repeats."""

   212         prev = self.dict.get(key)

   213         if prev is None:

   214             self.dict[key] = value

   215         else:

   216             combined = ", ".join((prev, value))

   217             self.dict[key] = combined

   218 

   219     def addcontinue(self, key, more):

   220         """Add more field data from a continuation line."""

   221         prev = self.dict[key]

   222         self.dict[key] = prev + "\n " + more

   223 

   224     def readheaders(self):

   225         """Read header lines.

   226 

   227         Read header lines up to the entirely blank line that terminates them.

   228         The (normally blank) line that ends the headers is skipped, but not

   229         included in the returned list.  If a non-header line ends the headers,

   230         (which is an error), an attempt is made to backspace over it; it is

   231         never included in the returned list.

   232 

   233         The variable self.status is set to the empty string if all went well,

   234         otherwise it is an error message.  The variable self.headers is a

   235         completely uninterpreted list of lines contained in the header (so

   236         printing them will reproduce the header exactly as it appears in the

   237         file).

   238 

   239         If multiple header fields with the same name occur, they are combined

   240         according to the rules in RFC 2616 sec 4.2:

   241 

   242         Appending each subsequent field-value to the first, each separated

   243         by a comma. The order in which header fields with the same field-name

   244         are received is significant to the interpretation of the combined

   245         field value.

   246         """

   247         # XXX The implementation overrides the readheaders() method of

   248         # rfc822.Message.  The base class design isn't amenable to

   249         # customized behavior here so the method here is a copy of the

   250         # base class code with a few small changes.

   251 

   252         self.dict = {}

   253         self.unixfrom = ''

   254         self.headers = hlist = []

   255         self.status = ''

   256         headerseen = ""

   257         firstline = 1

   258         startofline = unread = tell = None

   259         if hasattr(self.fp, 'unread'):

   260             unread = self.fp.unread

   261         elif self.seekable:

   262             tell = self.fp.tell

   263         while True:

   264             if tell:

   265                 try:

   266                     startofline = tell()

   267                 except IOError:

   268                     startofline = tell = None

   269                     self.seekable = 0

   270             line = self.fp.readline()

   271             if not line:

   272                 self.status = 'EOF in headers'

   273                 break

   274             # Skip unix From name time lines

   275             if firstline and line.startswith('From '):

   276                 self.unixfrom = self.unixfrom + line

   277                 continue

   278             firstline = 0

   279             if headerseen and line[0] in ' \t':

   280                 # XXX Not sure if continuation lines are handled properly

   281                 # for http and/or for repeating headers

   282                 # It's a continuation line.

   283                 hlist.append(line)

   284                 self.addcontinue(headerseen, line.strip())

   285                 continue

   286             elif self.iscomment(line):

   287                 # It's a comment.  Ignore it.

   288                 continue

   289             elif self.islast(line):

   290                 # Note! No pushback here!  The delimiter line gets eaten.

   291                 break

   292             headerseen = self.isheader(line)

   293             if headerseen:

   294                 # It's a legal header line, save it.

   295                 hlist.append(line)

   296                 self.addheader(headerseen, line[len(headerseen)+1:].strip())

   297                 continue

   298             else:

   299                 # It's not a header line; throw it back and stop here.

   300                 if not self.dict:

   301                     self.status = 'No headers'

   302                 else:

   303                     self.status = 'Non-header line where header expected'

   304                 # Try to undo the read.

   305                 if unread:

   306                     unread(line)

   307                 elif tell:

   308                     self.fp.seek(startofline)

   309                 else:

   310                     self.status = self.status + '; bad seek'

   311                 break

   312 

   313 class HTTPResponse:

   314 

   315     # strict: If true, raise BadStatusLine if the status line can't be

   316     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is

   317     # false because it prevents clients from talking to HTTP/0.9

   318     # servers.  Note that a response with a sufficiently corrupted

   319     # status line will look like an HTTP/0.9 response.

   320 

   321     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.

   322 

   323     def __init__(self, sock, debuglevel=0, strict=0, method=None):

   324         self.fp = sock.makefile('rb', 0)

   325         self.debuglevel = debuglevel

   326         self.strict = strict

   327         self._method = method

   328 

   329         self.msg = None

   330 

   331         # from the Status-Line of the response

   332         self.version = _UNKNOWN # HTTP-Version

   333         self.status = _UNKNOWN  # Status-Code

   334         self.reason = _UNKNOWN  # Reason-Phrase

   335 

   336         self.chunked = _UNKNOWN         # is "chunked" being used?

   337         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk

   338         self.length = _UNKNOWN          # number of bytes left in response

   339         self.will_close = _UNKNOWN      # conn will close at end of response

   340 

   341     def _read_status(self):

   342         # Initialize with Simple-Response defaults

   343         line = self.fp.readline()

   344         if self.debuglevel > 0:

   345             print "reply:", repr(line)

   346         if not line:

   347             # Presumably, the server closed the connection before

   348             # sending a valid response.

   349             raise BadStatusLine(line)

   350         try:

   351             [version, status, reason] = line.split(None, 2)

   352         except ValueError:

   353             try:

   354                 [version, status] = line.split(None, 1)

   355                 reason = ""

   356             except ValueError:

   357                 # empty version will cause next test to fail and status

   358                 # will be treated as 0.9 response.

   359                 version = ""

   360         if not version.startswith('HTTP/'):

   361             if self.strict:

   362                 self.close()

   363                 raise BadStatusLine(line)

   364             else:

   365                 # assume it's a Simple-Response from an 0.9 server

   366                 self.fp = LineAndFileWrapper(line, self.fp)

   367                 return "HTTP/0.9", 200, ""

   368 

   369         # The status code is a three-digit number

   370         try:

   371             status = int(status)

   372             if status < 100 or status > 999:

   373                 raise BadStatusLine(line)

   374         except ValueError:

   375             raise BadStatusLine(line)

   376         return version, status, reason

   377 

   378     def begin(self):

   379         if self.msg is not None:

   380             # we've already started reading the response

   381             return

   382 

   383         # read until we get a non-100 response

   384         while True:

   385             version, status, reason = self._read_status()

   386             if status != CONTINUE:

   387                 break

   388             # skip the header from the 100 response

   389             while True:

   390                 skip = self.fp.readline().strip()

   391                 if not skip:

   392                     break

   393                 if self.debuglevel > 0:

   394                     print "header:", skip

   395 

   396         self.status = status

   397         self.reason = reason.strip()

   398         if version == 'HTTP/1.0':

   399             self.version = 10

   400         elif version.startswith('HTTP/1.'):

   401             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1

   402         elif version == 'HTTP/0.9':

   403             self.version = 9

   404         else:

   405             raise UnknownProtocol(version)

   406 

   407         if self.version == 9:

   408             self.length = None

   409             self.chunked = 0

   410             self.will_close = 1

   411             self.msg = HTTPMessage(StringIO())

   412             return

   413 

   414         self.msg = HTTPMessage(self.fp, 0)

   415         if self.debuglevel > 0:

   416             for hdr in self.msg.headers:

   417                 print "header:", hdr,

   418 

   419         # don't let the msg keep an fp

   420         self.msg.fp = None

   421 

   422         # are we using the chunked-style of transfer encoding?

   423         tr_enc = self.msg.getheader('transfer-encoding')

   424         if tr_enc and tr_enc.lower() == "chunked":

   425             self.chunked = 1

   426             self.chunk_left = None

   427         else:

   428             self.chunked = 0

   429 

   430         # will the connection close at the end of the response?

   431         self.will_close = self._check_close()

   432 

   433         # do we have a Content-Length?

   434         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"

   435         length = self.msg.getheader('content-length')

   436         if length and not self.chunked:

   437             try:

   438                 self.length = int(length)

   439             except ValueError:

   440                 self.length = None

   441         else:

   442             self.length = None

   443 

   444         # does the body have a fixed length? (of zero)

   445         if (status == NO_CONTENT or status == NOT_MODIFIED or

   446             100 <= status < 200 or      # 1xx codes

   447             self._method == 'HEAD'):

   448             self.length = 0

   449 

   450         # if the connection remains open, and we aren't using chunked, and

   451         # a content-length was not provided, then assume that the connection

   452         # WILL close.

   453         if not self.will_close and \

   454            not self.chunked and \

   455            self.length is None:

   456             self.will_close = 1

   457 

   458     def _check_close(self):

   459         conn = self.msg.getheader('connection')

   460         if self.version == 11:

   461             # An HTTP/1.1 proxy is assumed to stay open unless

   462             # explicitly closed.

   463             conn = self.msg.getheader('connection')

   464             if conn and "close" in conn.lower():

   465                 return True

   466             return False

   467 

   468         # Some HTTP/1.0 implementations have support for persistent

   469         # connections, using rules different than HTTP/1.1.

   470 

   471         # For older HTTP, Keep-Alive indiciates persistent connection.

   472         if self.msg.getheader('keep-alive'):

   473             return False

   474 

   475         # At least Akamai returns a "Connection: Keep-Alive" header,

   476         # which was supposed to be sent by the client.

   477         if conn and "keep-alive" in conn.lower():

   478             return False

   479 

   480         # Proxy-Connection is a netscape hack.

   481         pconn = self.msg.getheader('proxy-connection')

   482         if pconn and "keep-alive" in pconn.lower():

   483             return False

   484 

   485         # otherwise, assume it will close

   486         return True

   487 

   488     def close(self):

   489         if self.fp:

   490             self.fp.close()

   491             self.fp = None

   492 

   493     def isclosed(self):

   494         # NOTE: it is possible that we will not ever call self.close(). This

   495         #       case occurs when will_close is TRUE, length is None, and we

   496         #       read up to the last byte, but NOT past it.

   497         #

   498         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be

   499         #          called, meaning self.isclosed() is meaningful.

   500         return self.fp is None

   501 

   502     # XXX It would be nice to have readline and __iter__ for this, too.

   503 

   504     def read(self, amt=None):

   505         if self.fp is None:

   506             return ''

   507 

   508         if self.chunked:

   509             return self._read_chunked(amt)

   510 

   511         if amt is None:

   512             # unbounded read

   513             if self.length is None:

   514                 s = self.fp.read()

   515             else:

   516                 s = self._safe_read(self.length)

   517                 self.length = 0

   518             self.close()        # we read everything

   519             return s

   520 

   521         if self.length is not None:

   522             if amt > self.length:

   523                 # clip the read to the "end of response"

   524                 amt = self.length

   525 

   526         # we do not use _safe_read() here because this may be a .will_close

   527         # connection, and the user is reading more bytes than will be provided

   528         # (for example, reading in 1k chunks)

   529         s = self.fp.read(amt)

   530         if self.length is not None:

   531             self.length -= len(s)

   532 

   533         return s

   534 

   535     def _read_chunked(self, amt):

   536         assert self.chunked != _UNKNOWN

   537         chunk_left = self.chunk_left

   538         value = ''

   539 

   540         # XXX This accumulates chunks by repeated string concatenation,

   541         # which is not efficient as the number or size of chunks gets big.

   542         while True:

   543             if chunk_left is None:

   544                 line = self.fp.readline()

   545                 i = line.find(';')

   546                 if i >= 0:

   547                     line = line[:i] # strip chunk-extensions

   548                 chunk_left = int(line, 16)

   549                 if chunk_left == 0:

   550                     break

   551             if amt is None:

   552                 value += self._safe_read(chunk_left)

   553             elif amt < chunk_left:

   554                 value += self._safe_read(amt)

   555                 self.chunk_left = chunk_left - amt

   556                 return value

   557             elif amt == chunk_left:

   558                 value += self._safe_read(amt)

   559                 self._safe_read(2)  # toss the CRLF at the end of the chunk

   560                 self.chunk_left = None

   561                 return value

   562             else:

   563                 value += self._safe_read(chunk_left)

   564                 amt -= chunk_left

   565 

   566             # we read the whole chunk, get another

   567             self._safe_read(2)      # toss the CRLF at the end of the chunk

   568             chunk_left = None

   569 

   570         # read and discard trailer up to the CRLF terminator

   571         ### note: we shouldn't have any trailers!

   572         while True:

   573             line = self.fp.readline()

   574             if not line:

   575                 # a vanishingly small number of sites EOF without

   576                 # sending the trailer

   577                 break

   578             if line == '\r\n':

   579                 break

   580 

   581         # we read everything; close the "file"

   582         self.close()

   583 

   584         return value

   585 

   586     def _safe_read(self, amt):

   587         """Read the number of bytes requested, compensating for partial reads.

   588 

   589         Normally, we have a blocking socket, but a read() can be interrupted

   590         by a signal (resulting in a partial read).

   591 

   592         Note that we cannot distinguish between EOF and an interrupt when zero

   593         bytes have been read. IncompleteRead() will be raised in this

   594         situation.

   595 

   596         This function should be used when <amt> bytes "should" be present for

   597         reading. If the bytes are truly not available (due to EOF), then the

   598         IncompleteRead exception can be used to detect the problem.

   599         """

   600         s = []

   601         while amt > 0:

   602             chunk = self.fp.read(min(amt, MAXAMOUNT))

   603             if not chunk:

   604                 raise IncompleteRead(s)

   605             s.append(chunk)

   606             amt -= len(chunk)

   607         return ''.join(s)

   608 

   609     def getheader(self, name, default=None):

   610         if self.msg is None:

   611             raise ResponseNotReady()

   612         return self.msg.getheader(name, default)

   613 

   614     def getheaders(self):

   615         """Return list of (header, value) tuples."""

   616         if self.msg is None:

   617             raise ResponseNotReady()

   618         return self.msg.items()

   619 

   620 

   621 class HTTPConnection:

   622 

   623     _http_vsn = 11

   624     _http_vsn_str = 'HTTP/1.1'

   625 

   626     response_class = HTTPResponse

   627     default_port = HTTP_PORT

   628     auto_open = 1

   629     debuglevel = 0

   630     strict = 0

   631 

   632     def __init__(self, host, port=None, strict=None):

   633         self.sock = None

   634         self._buffer = []

   635         self.__response = None

   636         self.__state = _CS_IDLE

   637         self._method = None

   638 

   639         self._set_hostport(host, port)

   640         if strict is not None:

   641             self.strict = strict

   642 

   643     def _set_hostport(self, host, port):

   644         if port is None:

   645             i = host.rfind(':')

   646             j = host.rfind(']')         # ipv6 addresses have [...]

   647             if i > j:

   648                 try:

   649                     port = int(host[i+1:])

   650                 except ValueError:

   651                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])

   652                 host = host[:i]

   653             else:

   654                 port = self.default_port

   655             if host and host[0] == '[' and host[-1] == ']':

   656                 host = host[1:-1]

   657         self.host = host

   658         self.port = port

   659 

   660     def set_debuglevel(self, level):

   661         self.debuglevel = level

   662 

   663     def connect(self):

   664         """Connect to the host and port specified in __init__."""

   665         msg = "getaddrinfo returns an empty list"

   666         for res in socket.getaddrinfo(self.host, self.port, 0,

   667                                       socket.SOCK_STREAM):

   668             af, socktype, proto, canonname, sa = res

   669             try:

   670                 self.sock = socket.socket(af, socktype, proto)

   671                 if self.debuglevel > 0:

   672                     print "connect: (%s, %s)" % (self.host, self.port)

   673                 self.sock.connect(sa)

   674             except socket.error, msg:

   675                 if self.debuglevel > 0:

   676                     print 'connect fail:', (self.host, self.port)

   677                 if self.sock:

   678                     self.sock.close()

   679                 self.sock = None

   680                 continue

   681             break

   682         if not self.sock:

   683             raise socket.error, msg

   684 

   685     def close(self):

   686         """Close the connection to the HTTP server."""

   687         if self.sock:

   688             self.sock.close()   # close it manually... there may be other refs

   689             self.sock = None

   690         if self.__response:

   691             self.__response.close()

   692             self.__response = None

   693         self.__state = _CS_IDLE

   694 

   695     def send(self, str):

   696         """Send `str' to the server."""

   697         if self.sock is None:

   698             if self.auto_open:

   699                 self.connect()

   700             else:

   701                 raise NotConnected()

   702 

   703         # send the data to the server. if we get a broken pipe, then close

   704         # the socket. we want to reconnect when somebody tries to send again.

   705         #

   706         # NOTE: we DO propagate the error, though, because we cannot simply

   707         #       ignore the error... the caller will know if they can retry.

   708         if self.debuglevel > 0:

   709             print "send:", repr(str)

   710         try:

   711             self.sock.sendall(str)

   712         except socket.error, v:

   713             if v[0] == 32:      # Broken pipe

   714                 self.close()

   715             raise

   716 

   717     def _output(self, s):

   718         """Add a line of output to the current request buffer.

   719 

   720         Assumes that the line does *not* end with \\r\\n.

   721         """

   722         self._buffer.append(s)

   723 

   724     def _send_output(self):

   725         """Send the currently buffered request and clear the buffer.

   726 

   727         Appends an extra \\r\\n to the buffer.

   728         """

   729         self._buffer.extend(("", ""))

   730         msg = "\r\n".join(self._buffer)

   731         del self._buffer[:]

   732         self.send(msg)

   733 

   734     def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):

   735         """Send a request to the server.

   736 

   737         `method' specifies an HTTP request method, e.g. 'GET'.

   738         `url' specifies the object being requested, e.g. '/index.html'.

   739         `skip_host' if True does not add automatically a 'Host:' header

   740         `skip_accept_encoding' if True does not add automatically an

   741            'Accept-Encoding:' header

   742         """

   743 

   744         # if a prior response has been completed, then forget about it.

   745         if self.__response and self.__response.isclosed():

   746             self.__response = None

   747 

   748 

   749         # in certain cases, we cannot issue another request on this connection.

   750         # this occurs when:

   751         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)

   752         #   2) a response to a previous request has signalled that it is going

   753         #      to close the connection upon completion.

   754         #   3) the headers for the previous response have not been read, thus

   755         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)

   756         #

   757         # if there is no prior response, then we can request at will.

   758         #

   759         # if point (2) is true, then we will have passed the socket to the

   760         # response (effectively meaning, "there is no prior response"), and

   761         # will open a new one when a new request is made.

   762         #

   763         # Note: if a prior response exists, then we *can* start a new request.

   764         #       We are not allowed to begin fetching the response to this new

   765         #       request, however, until that prior response is complete.

   766         #

   767         if self.__state == _CS_IDLE:

   768             self.__state = _CS_REQ_STARTED

   769         else:

   770             raise CannotSendRequest()

   771 

   772         # Save the method we use, we need it later in the response phase

   773         self._method = method

   774         if not url:

   775             url = '/'

   776         str = '%s %s %s' % (method, url, self._http_vsn_str)

   777 

   778         self._output(str)

   779 

   780         if self._http_vsn == 11:

   781             # Issue some standard headers for better HTTP/1.1 compliance

   782 

   783             if not skip_host:

   784                 # this header is issued *only* for HTTP/1.1

   785                 # connections. more specifically, this means it is

   786                 # only issued when the client uses the new

   787                 # HTTPConnection() class. backwards-compat clients

   788                 # will be using HTTP/1.0 and those clients may be

   789                 # issuing this header themselves. we should NOT issue

   790                 # it twice; some web servers (such as Apache) barf

   791                 # when they see two Host: headers

   792 

   793                 # If we need a non-standard port,include it in the

   794                 # header.  If the request is going through a proxy,

   795                 # but the host of the actual URL, not the host of the

   796                 # proxy.

   797 

   798                 netloc = ''

   799                 if url.startswith('http'):

   800                     nil, netloc, nil, nil, nil = urlsplit(url)

   801 

   802                 if netloc:

   803                     try:

   804                         netloc_enc = netloc.encode("ascii")

   805                     except UnicodeEncodeError:

   806                         netloc_enc = netloc.encode("idna")

   807                     self.putheader('Host', netloc_enc)

   808                 else:

   809                     try:

   810                         host_enc = self.host.encode("ascii")

   811                     except UnicodeEncodeError:

   812                         host_enc = self.host.encode("idna")

   813                     if self.port == HTTP_PORT:

   814                         self.putheader('Host', host_enc)

   815                     else:

   816                         self.putheader('Host', "%s:%s" % (host_enc, self.port))

   817 

   818             # note: we are assuming that clients will not attempt to set these

   819             #       headers since *this* library must deal with the

   820             #       consequences. this also means that when the supporting

   821             #       libraries are updated to recognize other forms, then this

   822             #       code should be changed (removed or updated).

   823 

   824             # we only want a Content-Encoding of "identity" since we don't

   825             # support encodings such as x-gzip or x-deflate.

   826             if not skip_accept_encoding:

   827                 self.putheader('Accept-Encoding', 'identity')

   828 

   829             # we can accept "chunked" Transfer-Encodings, but no others

   830             # NOTE: no TE header implies *only* "chunked"

   831             #self.putheader('TE', 'chunked')

   832 

   833             # if TE is supplied in the header, then it must appear in a

   834             # Connection header.

   835             #self.putheader('Connection', 'TE')

   836 

   837         else:

   838             # For HTTP/1.0, the server will assume "not chunked"

   839             pass

   840 

   841     def putheader(self, header, value):

   842         """Send a request header line to the server.

   843 

   844         For example: h.putheader('Accept', 'text/html')

   845         """

   846         if self.__state != _CS_REQ_STARTED:

   847             raise CannotSendHeader()

   848 

   849         str = '%s: %s' % (header, value)

   850         self._output(str)

   851 

   852     def endheaders(self):

   853         """Indicate that the last header line has been sent to the server."""

   854 

   855         if self.__state == _CS_REQ_STARTED:

   856             self.__state = _CS_REQ_SENT

   857         else:

   858             raise CannotSendHeader()

   859 

   860         self._send_output()

   861 

   862     def request(self, method, url, body=None, headers={}):

   863         """Send a complete request to the server."""

   864 

   865         try:

   866             self._send_request(method, url, body, headers)

   867         except socket.error, v:

   868             # trap 'Broken pipe' if we're allowed to automatically reconnect

   869             if v[0] != 32 or not self.auto_open:

   870                 raise

   871             # try one more time

   872             self._send_request(method, url, body, headers)

   873 

   874     def _send_request(self, method, url, body, headers):

   875         # honour explicitly requested Host: and Accept-Encoding headers

   876         header_names = dict.fromkeys([k.lower() for k in headers])

   877         skips = {}

   878         if 'host' in header_names:

   879             skips['skip_host'] = 1

   880         if 'accept-encoding' in header_names:

   881             skips['skip_accept_encoding'] = 1

   882 

   883         self.putrequest(method, url, **skips)

   884 

   885         if body and ('content-length' not in header_names):

   886             self.putheader('Content-Length', str(len(body)))

   887         for hdr, value in headers.iteritems():

   888             self.putheader(hdr, value)

   889         self.endheaders()

   890 

   891         if body:

   892             self.send(body)

   893 

   894     def getresponse(self):

   895         "Get the response from the server."

   896 

   897         # if a prior response has been completed, then forget about it.

   898         if self.__response and self.__response.isclosed():

   899             self.__response = None

   900 

   901         #

   902         # if a prior response exists, then it must be completed (otherwise, we

   903         # cannot read this response's header to determine the connection-close

   904         # behavior)

   905         #

   906         # note: if a prior response existed, but was connection-close, then the

   907         # socket and response were made independent of this HTTPConnection

   908         # object since a new request requires that we open a whole new

   909         # connection

   910         #

   911         # this means the prior response had one of two states:

   912         #   1) will_close: this connection was reset and the prior socket and

   913         #                  response operate independently

   914         #   2) persistent: the response was retained and we await its

   915         #                  isclosed() status to become true.

   916         #

   917         if self.__state != _CS_REQ_SENT or self.__response:

   918             raise ResponseNotReady()

   919 

   920         if self.debuglevel > 0:

   921             response = self.response_class(self.sock, self.debuglevel,

   922                                            strict=self.strict,

   923                                            method=self._method)

   924         else:

   925             response = self.response_class(self.sock, strict=self.strict,

   926                                            method=self._method)

   927 

   928         response.begin()

   929         assert response.will_close != _UNKNOWN

   930         self.__state = _CS_IDLE

   931 

   932         if response.will_close:

   933             # this effectively passes the connection to the response

   934             self.close()

   935         else:

   936             # remember this, so we can tell when it is complete

   937             self.__response = response

   938 

   939         return response

   940 

   941 # The next several classes are used to define FakeSocket, a socket-like

   942 # interface to an SSL connection.

   943 

   944 # The primary complexity comes from faking a makefile() method.  The

   945 # standard socket makefile() implementation calls dup() on the socket

   946 # file descriptor.  As a consequence, clients can call close() on the

   947 # parent socket and its makefile children in any order.  The underlying

   948 # socket isn't closed until they are all closed.

   949 

   950 # The implementation uses reference counting to keep the socket open

   951 # until the last client calls close().  SharedSocket keeps track of

   952 # the reference counting and SharedSocketClient provides an constructor

   953 # and close() method that call incref() and decref() correctly.

   954 

   955 class SharedSocket:

   956 

   957     def __init__(self, sock):

   958         self.sock = sock

   959         self._refcnt = 0

   960 

   961     def incref(self):

   962         self._refcnt += 1

   963 

   964     def decref(self):

   965         self._refcnt -= 1

   966         assert self._refcnt >= 0

   967         if self._refcnt == 0:

   968             self.sock.close()

   969 

   970     def __del__(self):

   971         self.sock.close()

   972 

   973 class SharedSocketClient:

   974 

   975     def __init__(self, shared):

   976         self._closed = 0

   977         self._shared = shared

   978         self._shared.incref()

   979         self._sock = shared.sock

   980 

   981     def close(self):

   982         if not self._closed:

   983             self._shared.decref()

   984             self._closed = 1

   985             self._shared = None

   986 

   987 class SSLFile(SharedSocketClient):

   988     """File-like object wrapping an SSL socket."""

   989 

   990     BUFSIZE = 8192

   991 

   992     def __init__(self, sock, ssl, bufsize=None):

   993         SharedSocketClient.__init__(self, sock)

   994         self._ssl = ssl

   995         self._buf = ''

   996         self._bufsize = bufsize or self.__class__.BUFSIZE

   997 

   998     def _read(self):

   999         buf = ''

  1000         # put in a loop so that we retry on transient errors

  1001         while True:

  1002             try:

  1003                 buf = self._ssl.read(self._bufsize)

  1004             except socket.sslerror, err:

  1005                 if (err[0] == socket.SSL_ERROR_WANT_READ

  1006                     or err[0] == socket.SSL_ERROR_WANT_WRITE):

  1007                     continue

  1008                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN

  1009                     or err[0] == socket.SSL_ERROR_EOF):

  1010                     break

  1011                 raise

  1012             except socket.error, err:

  1013                 if err[0] == errno.EINTR:

  1014                     continue

  1015                 if err[0] == errno.EBADF:

  1016                     # XXX socket was closed?

  1017                     break

  1018                 raise

  1019             else:

  1020                 break

  1021         return buf

  1022 

  1023     def read(self, size=None):

  1024         L = [self._buf]

  1025         avail = len(self._buf)

  1026         while size is None or avail < size:

  1027             s = self._read()

  1028             if s == '':

  1029                 break

  1030             L.append(s)

  1031             avail += len(s)

  1032         all = "".join(L)

  1033         if size is None:

  1034             self._buf = ''

  1035             return all

  1036         else:

  1037             self._buf = all[size:]

  1038             return all[:size]

  1039 

  1040     def readline(self):

  1041         L = [self._buf]

  1042         self._buf = ''

  1043         while 1:

  1044             i = L[-1].find("\n")

  1045             if i >= 0:

  1046                 break

  1047             s = self._read()

  1048             if s == '':

  1049                 break

  1050             L.append(s)

  1051         if i == -1:

  1052             # loop exited because there is no more data

  1053             return "".join(L)

  1054         else:

  1055             all = "".join(L)

  1056             # XXX could do enough bookkeeping not to do a 2nd search

  1057             i = all.find("\n") + 1

  1058             line = all[:i]

  1059             self._buf = all[i:]

  1060             return line

  1061 

  1062     def readlines(self, sizehint=0):

  1063         total = 0

  1064         list = []

  1065         while True:

  1066             line = self.readline()

  1067             if not line:

  1068                 break

  1069             list.append(line)

  1070             total += len(line)

  1071             if sizehint and total >= sizehint:

  1072                 break

  1073         return list

  1074 

  1075     def fileno(self):

  1076         return self._sock.fileno()

  1077 

  1078     def __iter__(self):

  1079         return self

  1080 

  1081     def next(self):

  1082         line = self.readline()

  1083         if not line:

  1084             raise StopIteration

  1085         return line

  1086 

  1087 class FakeSocket(SharedSocketClient):

  1088 

  1089     class _closedsocket:

  1090         def __getattr__(self, name):

  1091             raise error(9, 'Bad file descriptor')

  1092 

  1093     def __init__(self, sock, ssl):

  1094         sock = SharedSocket(sock)

  1095         SharedSocketClient.__init__(self, sock)

  1096         self._ssl = ssl

  1097 

  1098     def close(self):

  1099         SharedSocketClient.close(self)

  1100         self._sock = self.__class__._closedsocket()

  1101 

  1102     def makefile(self, mode, bufsize=None):

  1103         if mode != 'r' and mode != 'rb':

  1104             raise UnimplementedFileMode()

  1105         return SSLFile(self._shared, self._ssl, bufsize)

  1106 

  1107     def send(self, stuff, flags = 0):

  1108         return self._ssl.write(stuff)

  1109 

  1110     sendall = send

  1111 

  1112     def recv(self, len = 1024, flags = 0):

  1113         return self._ssl.read(len)

  1114 

  1115     def __getattr__(self, attr):

  1116         return getattr(self._sock, attr)

  1117 

  1118 

  1119 class HTTPSConnection(HTTPConnection):

  1120     "This class allows communication via SSL."

  1121 

  1122     default_port = HTTPS_PORT

  1123 

  1124     def __init__(self, host, port=None, key_file=None, cert_file=None,

  1125                  strict=None):

  1126         HTTPConnection.__init__(self, host, port, strict)

  1127         self.key_file = key_file

  1128         self.cert_file = cert_file

  1129 

  1130     def connect(self):

  1131         "Connect to a host on a given (SSL) port."

  1132 

  1133         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

  1134         sock.connect((self.host, self.port))

  1135         ssl = socket.ssl(sock, self.key_file, self.cert_file)