diff --git a/thirdparty/httplib2/__init__.py b/thirdparty/httplib2/__init__.py index 48a7d4d12..d0e1045b9 100644 --- a/thirdparty/httplib2/__init__.py +++ b/thirdparty/httplib2/__init__.py @@ -22,7 +22,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)", "Sam Ruby", "Louis Nyffenegger"] __license__ = "MIT" -__version__ = "0.7.4" +__version__ = "0.7.7" import re import sys @@ -111,6 +111,8 @@ __all__ = ['Http', 'Response', 'ProxyInfo', 'HttpLib2Error', # The httplib debug level, set to a non-zero value to get debug output debuglevel = 0 +# A request will be tried 'RETRIES' times if it fails at the socket/connection level. +RETRIES = 2 # Python 2.3 support if sys.version_info < (2,4): @@ -470,7 +472,7 @@ class Authentication(object): def request(self, method, request_uri, headers, content): """Modify the request headers to add the appropriate - Authorization header. Over-rise this in sub-classes.""" + Authorization header. Over-ride this in sub-classes.""" pass def response(self, response, content): @@ -761,67 +763,6 @@ class ProxyInfo(object): def isgood(self): return (self.proxy_host != None) and (self.proxy_port != None) - @classmethod - def from_environment(cls, method='http'): - """ - Read proxy info from the environment variables. - """ - if method not in ['http', 'https']: - return - - env_var = method + '_proxy' - url = os.environ.get(env_var, os.environ.get(env_var.upper())) - if not url: - return - pi = cls.from_url(url, method) - - no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) - bypass_hosts = [] - if no_proxy: - bypass_hosts = no_proxy.split(',') - # special case, no_proxy=* means all hosts bypassed - if no_proxy == '*': - bypass_hosts = AllHosts - - pi.bypass_hosts = bypass_hosts - return pi - - @classmethod - def from_url(cls, url, method='http'): - """ - Construct a ProxyInfo from a URL (such as http_proxy env var) - """ - url = urlparse.urlparse(url) - username = None - password = None - port = None - if '@' in url[1]: - ident, host_port = url[1].split('@', 1) - if ':' in ident: - username, password = ident.split(':', 1) - else: - password = ident - else: - host_port = url[1] - if ':' in host_port: - host, port = host_port.split(':', 1) - else: - host = host_port - - if port: - port = int(port) - else: - port = dict(https=443, http=80)[method] - - proxy_type = 3 # socks.PROXY_TYPE_HTTP - return cls( - proxy_type = proxy_type, - proxy_host = host, - proxy_port = port, - proxy_user = username or None, - proxy_pass = password or None, - ) - def applies_to(self, hostname): return not self.bypass_host(hostname) @@ -838,6 +779,66 @@ class ProxyInfo(object): return bypass +def proxy_info_from_environment(method='http'): + """ + Read proxy info from the environment variables. + """ + if method not in ['http', 'https']: + return + + env_var = method + '_proxy' + url = os.environ.get(env_var, os.environ.get(env_var.upper())) + if not url: + return + pi = proxy_info_from_url(url, method) + + no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) + bypass_hosts = [] + if no_proxy: + bypass_hosts = no_proxy.split(',') + # special case, no_proxy=* means all hosts bypassed + if no_proxy == '*': + bypass_hosts = AllHosts + + pi.bypass_hosts = bypass_hosts + return pi + +def proxy_info_from_url(url, method='http'): + """ + Construct a ProxyInfo from a URL (such as http_proxy env var) + """ + url = urlparse.urlparse(url) + username = None + password = None + port = None + if '@' in url[1]: + ident, host_port = url[1].split('@', 1) + if ':' in ident: + username, password = ident.split(':', 1) + else: + password = ident + else: + host_port = url[1] + if ':' in host_port: + host, port = host_port.split(':', 1) + else: + host = host_port + + if port: + port = int(port) + else: + port = dict(https=443, http=80)[method] + + proxy_type = 3 # socks.PROXY_TYPE_HTTP + return ProxyInfo( + proxy_type = proxy_type, + proxy_host = host, + proxy_port = port, + proxy_user = username or None, + proxy_pass = password or None, + ) + + class HTTPConnectionWithTimeout(httplib.HTTPConnection): """ HTTPConnection subclass that supports timeouts @@ -860,13 +861,24 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): raise ProxiesUnavailableError( 'Proxy support missing but proxy use was requested!') msg = "getaddrinfo returns an empty list" - for res in socket.getaddrinfo(self.host, self.port, 0, - socket.SOCK_STREAM): + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + + for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): af, socktype, proto, canonname, sa = res try: - if self.proxy_info and self.proxy_info.isgood(): + if use_proxy: self.sock = socks.socksocket(af, socktype, proto) - self.sock.setproxy(*self.proxy_info.astuple()) + self.sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) else: self.sock = socket.socket(af, socktype, proto) self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -875,12 +887,16 @@ class HTTPConnectionWithTimeout(httplib.HTTPConnection): self.sock.settimeout(self.timeout) # End of difference from httplib. if self.debuglevel > 0: - print "connect: (%s, %s)" % (self.host, self.port) + print "connect: (%s, %s) ************" % (self.host, self.port) + if use_proxy: + print "proxy: %s ************" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) self.sock.connect((self.host, self.port) + sa[2:]) except socket.error, msg: if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if self.sock: self.sock.close() self.sock = None @@ -966,12 +982,25 @@ class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): "Connect to a host on a given (SSL) port." msg = "getaddrinfo returns an empty list" + if self.proxy_info and self.proxy_info.isgood(): + use_proxy = True + proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass = self.proxy_info.astuple() + else: + use_proxy = False + if use_proxy and proxy_rdns: + host = proxy_host + port = proxy_port + else: + host = self.host + port = self.port + for family, socktype, proto, canonname, sockaddr in socket.getaddrinfo( - self.host, self.port, 0, socket.SOCK_STREAM): + host, port, 0, socket.SOCK_STREAM): try: - if self.proxy_info and self.proxy_info.isgood(): + if use_proxy: sock = socks.socksocket(family, socktype, proto) - sock.setproxy(*self.proxy_info.astuple()) + + sock.setproxy(proxy_type, proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass) else: sock = socket.socket(family, socktype, proto) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) @@ -984,6 +1013,8 @@ class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): self.disable_ssl_certificate_validation, self.ca_certs) if self.debuglevel > 0: print "connect: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if not self.disable_ssl_certificate_validation: cert = self.sock.getpeercert() hostname = self.host.split(':', 0)[0] @@ -1009,7 +1040,9 @@ class HTTPSConnectionWithTimeout(httplib.HTTPSConnection): raise except socket.error, msg: if self.debuglevel > 0: - print 'connect fail:', (self.host, self.port) + print "connect fail: (%s, %s)" % (self.host, self.port) + if use_proxy: + print "proxy: %s" % str((proxy_host, proxy_port, proxy_rdns, proxy_user, proxy_pass)) if self.sock: self.sock.close() self.sock = None @@ -1030,16 +1063,15 @@ try: raise ImportError # Bail out; we're not actually running on App Engine. from google.appengine.api.urlfetch import fetch from google.appengine.api.urlfetch import InvalidURLError - from google.appengine.api.urlfetch import DownloadError - from google.appengine.api.urlfetch import ResponseTooLargeError - from google.appengine.api.urlfetch import SSLCertificateError - class ResponseDict(dict): - """Is a dictionary that also has a read() method, so - that it can pass itself off as an httlib.HTTPResponse().""" + """Dictionary with a read() method; can pass off as httplib.HTTPResponse.""" + def __init__(self, *args, **kwargs): + self.content = kwargs.pop('content', None) + return super(ResponseDict, self).__init__(*args, **kwargs) + def read(self): - pass + return self.content class AppEngineHttpConnection(object): @@ -1050,7 +1082,7 @@ try: """ def __init__(self, host, port=None, key_file=None, cert_file=None, strict=None, timeout=None, proxy_info=None, ca_certs=None, - disable_certificate_validation=False): + disable_ssl_certificate_validation=False): self.host = host self.port = port self.timeout = timeout @@ -1058,7 +1090,7 @@ try: raise NotSupportedOnThisPlatform() self.response = None self.scheme = 'http' - self.validate_certificate = not disable_certificate_validation + self.validate_certificate = not disable_ssl_certificate_validation self.sock = True def request(self, method, url, body, headers): @@ -1068,20 +1100,22 @@ try: netloc = '%s:%s' % (self.host, self.port) absolute_uri = '%s://%s%s' % (self.scheme, netloc, url) try: + try: # 'body' can be a stream. + body = body.read() + except AttributeError: + pass response = fetch(absolute_uri, payload=body, method=method, headers=headers, allow_truncated=False, follow_redirects=False, deadline=self.timeout, validate_certificate=self.validate_certificate) - self.response = ResponseDict(response.headers) + self.response = ResponseDict(response.headers, content=response.content) self.response['status'] = str(response.status_code) + self.response['reason'] = httplib.responses.get(response.status_code, 'Ok') self.response.status = response.status_code - setattr(self.response, 'read', lambda : response.content) # Make sure the exceptions raised match the exceptions expected. except InvalidURLError: raise socket.gaierror('') - except (DownloadError, ResponseTooLargeError, SSLCertificateError): - raise httplib.HTTPException() def getresponse(self): if self.response: @@ -1102,9 +1136,10 @@ try: class AppEngineHttpsConnection(AppEngineHttpConnection): """Same as AppEngineHttpConnection, but for HTTPS URIs.""" def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=None, proxy_info=None): + strict=None, timeout=None, proxy_info=None, ca_certs=None, + disable_ssl_certificate_validation=False): AppEngineHttpConnection.__init__(self, host, port, key_file, cert_file, - strict, timeout, proxy_info) + strict, timeout, proxy_info, ca_certs, disable_ssl_certificate_validation) self.scheme = 'https' # Update the connection classes to use the Googel App Engine specific ones. @@ -1131,10 +1166,9 @@ class Http(object): and more. """ def __init__(self, cache=None, timeout=None, - proxy_info=ProxyInfo.from_environment, + proxy_info=proxy_info_from_environment, ca_certs=None, disable_ssl_certificate_validation=False): - """ - If 'cache' is a string then it is used as a directory name for + """If 'cache' is a string then it is used as a directory name for a disk cache. Otherwise it must be an object that supports the same interface as FileCache. @@ -1146,7 +1180,7 @@ and more. `proxy_info` may be: - a callable that takes the http scheme ('http' or 'https') and returns a ProxyInfo instance per request. By default, uses - ProxyInfo.from_environment. + proxy_nfo_from_environment. - a ProxyInfo instance (static proxy config). - None (proxy disabled). @@ -1197,6 +1231,23 @@ and more. self.timeout = timeout + # Keep Authorization: headers on a redirect. + self.forward_authorization_headers = False + + def __getstate__(self): + state_dict = copy.copy(self.__dict__) + # In case request is augmented by some foreign object such as + # credentials which handle auth + if 'request' in state_dict: + del state_dict['request'] + if 'connections' in state_dict: + del state_dict['connections'] + return state_dict + + def __setstate__(self, state): + self.__dict__.update(state) + self.connections = {} + def _auth_from_challenge(self, host, request_uri, headers, response, content): """A generator that creates Authorization objects that can be applied to requests. @@ -1224,7 +1275,7 @@ and more. self.authorizations = [] def _conn_request(self, conn, request_uri, method, body, headers): - for i in range(2): + for i in range(RETRIES): try: if conn.sock is None: conn.connect() @@ -1249,21 +1300,21 @@ and more. # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. if conn.sock is None: - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue else: conn.close() raise - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue try: response = conn.getresponse() except (socket.error, httplib.HTTPException): - if i == 0: + if i < RETRIES-1: conn.close() conn.connect() continue @@ -1272,7 +1323,7 @@ and more. else: content = "" if method == "HEAD": - response.close() + conn.close() else: content = response.read() response = Response(response) @@ -1330,6 +1381,8 @@ and more. del headers['if-none-match'] if headers.has_key('if-modified-since'): del headers['if-modified-since'] + if 'authorization' in headers and not self.forward_authorization_headers: + del headers['authorization'] if response.has_key('location'): location = response['location'] old_response = copy.deepcopy(response) @@ -1406,7 +1459,7 @@ a string that contains the response entity body. if not connection_type: connection_type = SCHEME_TO_CONNECTION[scheme] certs = list(self.certificates.iter(authority)) - if issubclass(connection_type, HTTPSConnectionWithTimeout): + if scheme == 'https': if certs: conn = self.connections[conn_key] = connection_type( authority, key_file=certs[0][0], @@ -1448,7 +1501,7 @@ a string that contains the response entity body. feedparser.feed(info) info = feedparser.close() feedparser._parse = None - except IndexError: + except (IndexError, ValueError): self.cache.delete(cachekey) cachekey = None cached_value = None @@ -1615,12 +1668,13 @@ class Response(dict): self.version = info.version elif isinstance(info, email.Message.Message): for key, value in info.items(): - self[key] = value + self[key.lower()] = value self.status = int(self['status']) else: for key, value in info.iteritems(): - self[key] = value + self[key.lower()] = value self.status = int(self.get('status', self.status)) + self.reason = self.get('reason', self.reason) def __getattr__(self, name): diff --git a/thirdparty/httplib2/cacerts.txt b/thirdparty/httplib2/cacerts.txt index da36ed1ba..d8a0027cc 100644 --- a/thirdparty/httplib2/cacerts.txt +++ b/thirdparty/httplib2/cacerts.txt @@ -712,3 +712,28 @@ IYEZoDJJKPTEjlbVUjP9UNV+mWwD5MlM/Mtsq2azSiGM5bUMMj4QssxsodyamEwC W/POuZ6lcg5Ktz885hZo+L7tdEy8W9ViH0Pd -----END CERTIFICATE----- +GeoTrust Global CA +================== + +-----BEGIN CERTIFICATE----- +MIIDfTCCAuagAwIBAgIDErvmMA0GCSqGSIb3DQEBBQUAME4xCzAJBgNVBAYTAlVT +MRAwDgYDVQQKEwdFcXVpZmF4MS0wKwYDVQQLEyRFcXVpZmF4IFNlY3VyZSBDZXJ0 +aWZpY2F0ZSBBdXRob3JpdHkwHhcNMDIwNTIxMDQwMDAwWhcNMTgwODIxMDQwMDAw +WjBCMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UE +AxMSR2VvVHJ1c3QgR2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEA2swYYzD99BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9m +OSm9BXiLnTjoBbdqfnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIu +T8rxh0PBFpVXLVDviS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6c +JmTM386DGXHKTubU1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmR +Cw7+OC7RHQWa9k0+bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5asz +PeE4uwc2hGKceeoWMPRfwCvocWvk+QIDAQABo4HwMIHtMB8GA1UdIwQYMBaAFEjm +aPkr0rKV10fYIyAQTzOYkJ/UMB0GA1UdDgQWBBTAephojYn7qwVkDBF9qn1luMrM +TjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjA6BgNVHR8EMzAxMC+g +LaArhilodHRwOi8vY3JsLmdlb3RydXN0LmNvbS9jcmxzL3NlY3VyZWNhLmNybDBO +BgNVHSAERzBFMEMGBFUdIAAwOzA5BggrBgEFBQcCARYtaHR0cHM6Ly93d3cuZ2Vv +dHJ1c3QuY29tL3Jlc291cmNlcy9yZXBvc2l0b3J5MA0GCSqGSIb3DQEBBQUAA4GB +AHbhEm5OSxYShjAGsoEIz/AIx8dxfmbuwu3UOx//8PDITtZDOLC5MH0Y0FWDomrL +NhGc6Ehmo21/uBPUR/6LWlxz/K7ZGzIZOKuXNBSqltLroxwUCEm2u+WR74M26x1W +b8ravHNjkOR/ez4iyz0H7V84dJzjA1BOoa+Y7mHyhD8S +-----END CERTIFICATE----- + diff --git a/thirdparty/httplib2/socks.py b/thirdparty/httplib2/socks.py index d21a5e2b9..0991f4cf6 100644 --- a/thirdparty/httplib2/socks.py +++ b/thirdparty/httplib2/socks.py @@ -43,6 +43,7 @@ mainly to merge bug fixes found in Sourceforge import base64 import socket import struct +import sys if getattr(socket, 'socket', None) is None: raise ImportError('socket.socket missing, proxy support unusable')