I applied the recent changes to the CVS httplib to Greg's httplib
(call it httplib11) this afternoon. The result is included below. I
think this is quite close to checking in, but it could use a slightly
better test suite.
There are a few outstanding questions.
httplib11 does not implement the debuglevel feature. I don't think
it's important, but it is currently documented and may be used.
Guido, should we implement it?
httplib w/SSL uses a constructor with this prototype:
def __init__(self, host='', port=None, **x509):
It looks like the x509 dictionary should contain two variables --
key_file and cert_file. Since we know what the entries are, why not
make them explicit?
def __init__(self, host='', port=None, cert_file=None, key_file=None):
(Or reverse the two arguments if that is clearer.)
The FakeSocket class in CVS has a comment after the makefile def line
that says "hopefully, never have to write." It won't do at all the
right thing when called with a write mode, so it ought to raise an
exception. Any reason it doesn't?
I'd like to add a couple of test cases that use HTTP/1.1 to get some
pages from python.org, including one that uses the chunked encoding.
Just haven't gotten around to it. Question on that front: Does it
make sense to incorporate the test function in the module with the std
regression test suite? In general, I would think so. In this
particular case, the test could fail because of host networking
problems. I think that's okay as long as the error message is clear
enough.
Jeremy
"""HTTP/1.1 client library"""
# Written by Greg Stein.
import socket
import string
import mimetools
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
error = 'httplib.error'
HTTP_PORT = 80
HTTPS_PORT = 443
class HTTPResponse(mimetools.Message):
__super_init = mimetools.Message.__init__
def __init__(self, fp, version, errcode):
self.__super_init(fp, 0)
if version == 'HTTP/1.0':
self.version = 10
elif version[:7] == 'HTTP/1.':
self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
else:
raise error, 'unknown HTTP protocol'
# are we using the chunked-style of transfer encoding?
tr_enc = self.getheader('transfer-encoding')
if tr_enc:
if string.lower(tr_enc) != 'chunked':
raise error, 'unknown transfer-encoding'
self.chunked = 1
self.chunk_left = None
else:
self.chunked = 0
# will the connection close at the end of the response?
conn = self.getheader('connection')
if conn:
conn = string.lower(conn)
# a "Connection: close" will always close the
# connection. if we don't see that and this is not
# HTTP/1.1, then the connection will close unless we see a
# Keep-Alive header.
self.will_close = string.find(conn, 'close') != -1 or \
( self.version != 11 and \
not self.getheader('keep-alive') )
else:
# for HTTP/1.1, the connection will always remain open
# otherwise, it will remain open IFF we see a Keep-Alive header
self.will_close = self.version != 11 and \
not self.getheader('keep-alive')
# do we have a Content-Length?
# NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
length = self.getheader('content-length')
if length and not self.chunked:
self.length = int(length)
else:
self.length = None
# does the body have a fixed length? (of zero)
if (errcode == 204 or # No Content
errcode == 304 or # Not Modified
100 <= errcode < 200): # 1xx codes
self.length = 0
# if the connection remains open, and we aren't using chunked, and
# a content-length was not provided, then assume that the connection
# WILL close.
if not self.will_close and \
not self.chunked and \
self.length is None:
self.will_close = 1
# if there is no body, then close NOW. read() may never be
# called, thus we will never mark self as closed.
if self.length == 0:
self.close()
def close(self):
if self.fp:
self.fp.close()
self.fp = None
def isclosed(self):
# NOTE: it is possible that we will not ever call self.close(). This
# case occurs when will_close is TRUE, length is None, and we
# read up to the last byte, but NOT past it.
#
# IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
# called, meaning self.isclosed() is meaningful.
return self.fp is None
def read(self, amt=None):
if self.fp is None:
return ''
if self.chunked:
chunk_left = self.chunk_left
value = ''
while 1:
if chunk_left is None:
line = self.fp.readline()
i = string.find(line, ';')
if i >= 0:
line = line[:i] # strip chunk-extensions
chunk_left = string.atoi(line, 16)
if chunk_left == 0:
break
if amt is None:
value = value + self.fp.read(chunk_left)
elif amt < chunk_left:
value = value + self.fp.read(amt)
self.chunk_left = chunk_left - amt
return value
elif amt == chunk_left:
value = value + self.fp.read(amt)
self.fp.read(2) # toss the CRLF at the end of the chunk
self.chunk_left = None
return value
else:
value = value + self.fp.read(chunk_left)
amt = amt - chunk_left
# we read the whole chunk, get another
self.fp.read(2) # toss the CRLF at the end of the chunk
chunk_left = None
# read and discard trailer up to the CRLF terminator
### note: we shouldn't have any trailers!
while 1:
line = self.fp.readline()
if line == '\r\n':
break
# we read everything; close the "file"
self.close()
return value
elif amt is None:
# unbounded read
if self.will_close:
s = self.fp.read()
else:
s = self.fp.read(self.length)
self.close() # we read everything
return s
if self.length is not None:
if amt > self.length:
# clip the read to the "end of response"
amt = self.length
self.length = self.length - amt
s = self.fp.read(amt)
# close our "file" if we know we should
### I'm not sure about the len(s) < amt part; we should be
### safe because we shouldn't be using non-blocking sockets
if self.length == 0 or len(s) < amt:
self.close()
return s
class HTTPConnection:
_http_vsn = 11
_http_vsn_str = 'HTTP/1.1'
response_class = HTTPResponse
default_port = HTTP_PORT
def __init__(self, host, port=None):
self.sock = None
self.response = None
self._set_hostport(host, port)
def _set_hostport(self, host, port):
if port is None:
i = string.find(host, ':')
if i >= 0:
port = int(host[i+1:])
host = host[:i]
else:
port = self.default_port
self.host = host
self.port = port
self.addr = host, port
def connect(self):
"""Connect to the host and port specified in __init__."""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect(self.addr)
def close(self):
"""Close the connection to the HTTP server."""
if self.sock:
self.sock.close() # close it manually... there may be other refs
self.sock = None
if self.response:
self.response.close()
self.response = None
def send(self, str):
"""Send `str' to the server."""
if self.sock is None:
self.connect()
# send the data to the server. if we get a broken pipe, then close
# the socket. we want to reconnect when somebody tries to send again.
#
# NOTE: we DO propagate the error, though, because we cannot simply
# ignore the error... the caller will know if they can retry.
try:
self.sock.send(str)
except socket.error, v:
if v[0] == 32: # Broken pipe
self.close()
raise
def putrequest(self, method, url):
"""Send a request to the server.
`method' specifies an HTTP request method, e.g. 'GET'.
`url' specifies the object being requested, e.g.
'/index.html'.
"""
if self.response is not None:
if not self.response.isclosed():
### implies half-duplex!
raise error, 'prior response has not been fully handled'
self.response = None
if not url:
url = '/'
str = '%s %s %s\r\n' % (method, url, self._http_vsn_str)
try:
self.send(str)
except socket.error, v:
if v[0] != 32: # Broken pipe
raise
# try one more time (the socket was closed; this will reopen)
self.send(str)
self.putheader('Host', self.host)
if self._http_vsn == 11:
# Issue some standard headers for better HTTP/1.1 compliance
# note: we are assuming that clients will not attempt to set these
# headers since *this* library must deal with the consequences.
# this also means that when the supporting libraries are
# updated to recognize other forms, then this code should be
# changed (removed or updated).
# we only want a Content-Encoding of "identity" since we don't
# support encodings such as x-gzip or x-deflate.
self.putheader('Accept-Encoding', 'identity')
# we can accept "chunked" Transfer-Encodings, but no others
# NOTE: no TE header implies *only* "chunked"
#self.putheader('TE', 'chunked')
# if TE is supplied in the header, then it must appear in a
# Connection header.
#self.putheader('Connection', 'TE')
else:
# For HTTP/1.0, the server will assume "not chunked"
pass
def putheader(self, header, value):
"""Send a request header line to the server.
For example: h.putheader('Accept', 'text/html')
"""
str = '%s: %s\r\n' % (header, value)
self.send(str)
def endheaders(self):
"""Indicate that the last header line has been sent to the server."""
self.send('\r\n')
def request(self, method, url, body=None, headers={}):
"""Send a complete request to the server."""
try:
self._send_request(method, url, body, headers)
except socket.error, v:
if v[0] != 32: # Broken pipe
raise
# try one more time
self._send_request(method, url, body, headers)
def _send_request(self, method, url, body, headers):
self.putrequest(method, url)
if body:
self.putheader('Content-Length', str(len(body)))
for hdr, value in headers.items():
self.putheader(hdr, value)
self.endheaders()
if body:
self.send(body)
def getreply(self):
"""Get a reply from the server.
Returns a tuple consisting of:
- server response code (e.g. '200' if all goes well)
- server response string corresponding to response code
- any RFC822 headers in the response from the server
"""
file = self.sock.makefile('rb')
line = file.readline()
try:
[ver, code, msg] = string.split(line, None, 2)
except ValueError:
try:
[ver, code] = string.split(line, None, 1)
msg = ""
except ValueError:
self.close()
return -1, line, file
if ver[:5] != 'HTTP/':
self.close()
return -1, line, file
errcode = int(code)
errmsg = string.strip(msg)
response = self.response_class(file, ver, errcode)
if response.will_close:
# this effectively passes the connection to the response
self.close()
else:
# remember this, so we can tell when it is complete
self.response = response
return errcode, errmsg, response
class FakeSocket:
def __init__(self, sock, ssl):
self.__sock = sock
self.__ssl = ssl
return
def makefile(self, mode): # hopefully, never have to write
# XXX add assert about mode != w???
msgbuf = ""
while 1:
try:
msgbuf = msgbuf + self.__ssl.read()
except socket.sslerror, msg:
break
return StringIO(msgbuf)
def send(self, stuff, flags = 0):
return self.__ssl.write(stuff)
def recv(self, len = 1024, flags = 0):
return self.__ssl.read(len)
def __getattr__(self, attr):
return getattr(self.__sock, attr)
class HTTPSConnection(HTTPConnection):
"""This class allows communication via SSL."""
__super_init = HTTPConnection.__init__
default_port = HTTPS_PORT
def __init__(self, host, port=None, **x509):
self.__super_init(host, port)
self.key_file = x509.get('key_file')
self.cert_file = x509.get('cert_file')
def connect(self):
"""Connect to a host onf a given port
Note: This method is automatically invoked by __init__, if a host
is specified during instantiation.
"""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(self.addr)
ssl = socket.ssl(sock, self.key_file, self.cert_file)
self.sock = FakeSocket(sock, ssl)
class HTTPMixin:
"""Mixin for compatibility with httplib.py from 1.5.
requires that class that inherits defines the following attributes:
super_init
super_connect
super_putheader
super_getreply
"""
_http_vsn = 10
_http_vsn_str = 'HTTP/1.0'
def connect(self, host=None, port=None):
"Accept arguments to set the host/port, since the superclass doesn't."
if host is not None:
self._set_hostport(host, port)
self.super_connect()
def set_debuglevel(self, debuglevel):
"The class no longer supports the debuglevel."
pass
def getfile(self):
"Provide a getfile, since the superclass' use of HTTP/1.1 prevents it."
return self.file
def putheader(self, header, *values):
"The superclass allows only one value argument."
self.super_putheader(header, string.joinfields(values,'\r\n\t'))
def getreply(self):
"Compensate for an instance attribute shuffling."
errcode, errmsg, response = self.super_getreply()
if errcode == -1:
self.file = response # response is the "file" when errcode==-1
self.headers = None
return -1, errmsg, None
self.headers = response
self.file = response.fp
return errcode, errmsg, response
class HTTP(HTTPMixin, HTTPConnection):
super_init = HTTPConnection.__init__
super_connect = HTTPConnection.connect
super_putheader = HTTPConnection.putheader
super_getreply = HTTPConnection.getreply
_http_vsn = 10
_http_vsn_str = 'HTTP/1.0'
def __init__(self, host='', port=None):
"Provide a default host, since the superclass requires one."
# Note that we may pass an empty string as the host; this will throw
# an error when we attempt to connect. Presumably, the client code
# will call connect before then, with a proper host.
self.super_init(host, port)
class HTTPS(HTTPMixin, HTTPSConnection):
super_init = HTTPSConnection.__init__
super_connect = HTTPSConnection.connect
super_putheader = HTTPSConnection.putheader
super_getreply = HTTPSConnection.getreply
_http_vsn = 10
_http_vsn_str = 'HTTP/1.0'
def __init__(self, host='', port=None, **x509):
"Provide a default host, since the superclass requires one."
# Note that we may pass an empty string as the host; this will throw
# an error when we attempt to connect. Presumably, the client code
# will call connect before then, with a proper host.
self.super_init(host, port, **x509)
def test():
"""Test this module.
The test consists of retrieving and displaying the Python
home page, along with the error code and error string returned
by the www.python.org server.
"""
import sys
import getopt
opts, args = getopt.getopt(sys.argv[1:], 'd')
dl = 0
for o, a in opts:
if o == '-d': dl = dl + 1
host = 'www.python.org'
selector = '/'
if args[0:]: host = args[0]
if args[1:]: selector = args[1]
h = HTTP()
h.set_debuglevel(dl)
h.connect(host)
h.putrequest('GET', selector)
h.endheaders()
errcode, errmsg, headers = h.getreply()
print 'errcode =', errcode
print 'errmsg =', errmsg
print
if headers:
for header in headers.headers: print string.strip(header)
print
print h.getfile().read()
if hasattr(socket, 'ssl'):
host = 'www.c2.net'
hs = HTTPS()
hs.connect(host)
hs.putrequest('GET', selector)
hs.endheaders()
errcode, errmsg, headers = hs.getreply()
print 'errcode =', errcode
print 'errmsg =', errmsg
print
if headers:
for header in headers.headers: print string.strip(header)
print
print hs.getfile().read()
if __name__ == '__main__':
test()
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4