From 53f26c499faa4f1150c48da746c8c449549f3d21 Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 11:43:11 -0300 Subject: [PATCH 1/8] Started by copying over paste.proxy.Proxy --- restkit/contrib/wsgi_proxy.py | 135 ++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index 2e7d51ef..05bccba6 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -169,3 +169,138 @@ def make_host_proxy(global_config, uri=None, **local_config): uri = uri.rstrip('/') config = get_config(local_config) return HostProxy(uri, **config) + + + +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php + + +import httplib +import urlparse +import urllib + +from paste import httpexceptions +from paste.util.converters import aslist + +# Remove these headers from response (specify lower case header +# names): +filtered_headers = ( + 'transfer-encoding', + 'connection', + 'keep-alive', + 'proxy-authenticate', + 'proxy-authorization', + 'te', + 'trailers', + 'upgrade', +) + +class PasteLikeProxy(object): + + def __init__(self, address, allowed_request_methods=(), + suppress_http_headers=(), **kwargs): + self.address = address + self.parsed = urlparse.urlsplit(address) + self.scheme = self.parsed[0].lower() + self.host = self.parsed[1] + self.path = self.parsed[2] + self.allowed_request_methods = [ + x.lower() for x in allowed_request_methods if x] + + self.suppress_http_headers = [ + x.lower() for x in suppress_http_headers if x] + + def __call__(self, environ, start_response): + if (self.allowed_request_methods and + environ['REQUEST_METHOD'].lower() not in self.allowed_request_methods): + return httpexceptions.HTTPBadRequest("Disallowed")(environ, start_response) + + if self.scheme == 'http': + ConnClass = httplib.HTTPConnection + elif self.scheme == 'https': + ConnClass = httplib.HTTPSConnection + else: + raise ValueError( + "Unknown scheme for %r: %r" % (self.address, self.scheme)) + conn = ConnClass(self.host) + headers = {} + for key, value in environ.items(): + if key.startswith('HTTP_'): + key = key[5:].lower().replace('_', '-') + if key == 'host' or key in self.suppress_http_headers: + continue + headers[key] = value + headers['host'] = self.host + if 'REMOTE_ADDR' in environ: + headers['x-forwarded-for'] = environ['REMOTE_ADDR'] + if environ.get('CONTENT_TYPE'): + headers['content-type'] = environ['CONTENT_TYPE'] + if environ.get('CONTENT_LENGTH'): + if environ['CONTENT_LENGTH'] == '-1': + # This is a special case, where the content length is basically undetermined + body = environ['wsgi.input'].read(-1) + headers['content-length'] = str(len(body)) + else: + headers['content-length'] = environ['CONTENT_LENGTH'] + length = int(environ['CONTENT_LENGTH']) + body = environ['wsgi.input'].read(length) + else: + body = '' + + path_info = urllib.quote(environ['PATH_INFO']) + if self.path: + request_path = path_info + if request_path and request_path[0] == '/': + request_path = request_path[1:] + + path = urlparse.urljoin(self.path, request_path) + else: + path = path_info + if environ.get('QUERY_STRING'): + path += '?' + environ['QUERY_STRING'] + + conn.request(environ['REQUEST_METHOD'], + path, + body, headers) + res = conn.getresponse() + headers_out = parse_headers(res.msg) + + status = '%s %s' % (res.status, res.reason) + start_response(status, headers_out) + # @@: Default? + length = res.getheader('content-length') + if length is not None: + body = res.read(int(length)) + else: + body = res.read() + conn.close() + return [body] + + +def parse_headers(message): + """ + Turn a Message object into a list of WSGI-style headers. + """ + headers_out = [] + for full_header in message.headers: + if not full_header: + # Shouldn't happen, but we'll just ignore + continue + if full_header[0].isspace(): + # Continuation line, add to the last header + if not headers_out: + raise ValueError( + "First header starts with a space (%r)" % full_header) + last_header, last_value = headers_out.pop() + value = last_value + ' ' + full_header.strip() + headers_out.append((last_header, value)) + continue + try: + header, value = full_header.split(':', 1) + except: + raise ValueError("Invalid header: %r" % full_header) + value = value.strip() + if header.lower() not in filtered_headers: + headers_out.append((header, value)) + return headers_out From 077b9355f45a544430f961bfc5f2f4664c4d31ca Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 21:46:49 -0300 Subject: [PATCH 2/8] Paste's Proxy using restkit requests stack --- restkit/contrib/wsgi_proxy.py | 51 ++++++++++------------------------- 1 file changed, 14 insertions(+), 37 deletions(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index 05bccba6..c74f755e 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -211,19 +211,14 @@ def __init__(self, address, allowed_request_methods=(), self.suppress_http_headers = [ x.lower() for x in suppress_http_headers if x] + self.client = Client(**kwargs) + def __call__(self, environ, start_response): if (self.allowed_request_methods and environ['REQUEST_METHOD'].lower() not in self.allowed_request_methods): return httpexceptions.HTTPBadRequest("Disallowed")(environ, start_response) - if self.scheme == 'http': - ConnClass = httplib.HTTPConnection - elif self.scheme == 'https': - ConnClass = httplib.HTTPSConnection - else: - raise ValueError( - "Unknown scheme for %r: %r" % (self.address, self.scheme)) - conn = ConnClass(self.host) + conn = self.client headers = {} for key, value in environ.items(): if key.startswith('HTTP_'): @@ -260,47 +255,29 @@ def __call__(self, environ, start_response): if environ.get('QUERY_STRING'): path += '?' + environ['QUERY_STRING'] - conn.request(environ['REQUEST_METHOD'], - path, - body, headers) - res = conn.getresponse() - headers_out = parse_headers(res.msg) + res = conn.request(u'%s://%s%s' % (self.scheme, self.host, path), + environ['REQUEST_METHOD'], + body=body, headers=headers) + headers_out = parse_headers(res.headerslist) - status = '%s %s' % (res.status, res.reason) + status = res.status start_response(status, headers_out) # @@: Default? - length = res.getheader('content-length') + length = res.headers.get('Content-Length') if length is not None: - body = res.read(int(length)) + body = res.tee().read(int(length)) else: - body = res.read() - conn.close() + body = res.tee().read() + res.close() return [body] -def parse_headers(message): +def parse_headers(headers_list): """ Turn a Message object into a list of WSGI-style headers. """ headers_out = [] - for full_header in message.headers: - if not full_header: - # Shouldn't happen, but we'll just ignore - continue - if full_header[0].isspace(): - # Continuation line, add to the last header - if not headers_out: - raise ValueError( - "First header starts with a space (%r)" % full_header) - last_header, last_value = headers_out.pop() - value = last_value + ' ' + full_header.strip() - headers_out.append((last_header, value)) - continue - try: - header, value = full_header.split(':', 1) - except: - raise ValueError("Invalid header: %r" % full_header) - value = value.strip() + for header, value in headers_list: if header.lower() not in filtered_headers: headers_out.append((header, value)) return headers_out From 89400ef266df64752720e512dcc663d31c1ef1f0 Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 23:09:14 -0300 Subject: [PATCH 3/8] whitespace --- restkit/contrib/wsgi_proxy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index c74f755e..be18c241 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -231,6 +231,7 @@ def __call__(self, environ, start_response): headers['x-forwarded-for'] = environ['REMOTE_ADDR'] if environ.get('CONTENT_TYPE'): headers['content-type'] = environ['CONTENT_TYPE'] + if environ.get('CONTENT_LENGTH'): if environ['CONTENT_LENGTH'] == '-1': # This is a special case, where the content length is basically undetermined From 11cc52faa7d7463d9275721733cc737d0c15118e Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 23:10:19 -0300 Subject: [PATCH 4/8] Allow streaming, following the PEP 333 #handling-the-content-length-header See: http://www.python.org/dev/peps/pep-0333/#handling-the-content-length-header --- restkit/contrib/wsgi_proxy.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index be18c241..f01c60da 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -199,7 +199,7 @@ def make_host_proxy(global_config, uri=None, **local_config): class PasteLikeProxy(object): def __init__(self, address, allowed_request_methods=(), - suppress_http_headers=(), **kwargs): + suppress_http_headers=(), stream=False, **kwargs): self.address = address self.parsed = urlparse.urlsplit(address) self.scheme = self.parsed[0].lower() @@ -211,6 +211,7 @@ def __init__(self, address, allowed_request_methods=(), self.suppress_http_headers = [ x.lower() for x in suppress_http_headers if x] + self.stream = stream self.client = Client(**kwargs) def __call__(self, environ, start_response): @@ -259,26 +260,37 @@ def __call__(self, environ, start_response): res = conn.request(u'%s://%s%s' % (self.scheme, self.host, path), environ['REQUEST_METHOD'], body=body, headers=headers) - headers_out = parse_headers(res.headerslist) + headers_out = parse_headers(res.headerslist, stream=self.stream) status = res.status start_response(status, headers_out) # @@: Default? - length = res.headers.get('Content-Length') - if length is not None: - body = res.tee().read(int(length)) + if self.stream: + # See: http://www.python.org/dev/peps/pep-0333/#handling-the-content-length-header + body = res.body_stream() else: - body = res.tee().read() - res.close() - return [body] + length = res.headers.get('Content-Length') + if length is not None: + body = res.tee().read(int(length)) + else: + body = res.tee().read() + body = [body] + res.close() + return body -def parse_headers(headers_list): +def parse_headers(headers_list, stream=False): """ Turn a Message object into a list of WSGI-style headers. """ headers_out = [] for header, value in headers_list: + if stream: + # Suppress 'content-length' header: + # - The WSGI server CAN stream the response, if possible + # See: http://www.python.org/dev/peps/pep-0333/#handling-the-content-length-header + if header.lower() == 'content-length': + continue if header.lower() not in filtered_headers: headers_out.append((header, value)) return headers_out From 31afd7474e0cf759af5126f84bc75ec10ea9bae1 Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 23:12:02 -0300 Subject: [PATCH 5/8] Use ResponseTeeInput to guard against big response --- restkit/contrib/wsgi_proxy.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index f01c60da..dbc2f746 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -269,13 +269,7 @@ def __call__(self, environ, start_response): # See: http://www.python.org/dev/peps/pep-0333/#handling-the-content-length-header body = res.body_stream() else: - length = res.headers.get('Content-Length') - if length is not None: - body = res.tee().read(int(length)) - else: - body = res.tee().read() - body = [body] - res.close() + body = res.tee() return body From fa6f1e8781e975f92687e65922bce8fa907de3e7 Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 23:38:03 -0300 Subject: [PATCH 6/8] [FIX] stream=False still streams --- restkit/contrib/wsgi_proxy.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index dbc2f746..3717932f 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -267,9 +267,18 @@ def __call__(self, environ, start_response): # @@: Default? if self.stream: # See: http://www.python.org/dev/peps/pep-0333/#handling-the-content-length-header - body = res.body_stream() + if self.stream == 'safe': + body = res.tee() + else: + body = res.body_stream() else: - body = res.tee() + length = res.headers.get('Content-Length') + if length is not None: + body = res.body_string()[:int(length)] + else: + body = res.body_string() + body = [body] + res.close() return body From 8608fa6adf2f0540298160b4f8059fe631bb75c5 Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Sun, 28 Apr 2013 23:51:17 -0300 Subject: [PATCH 7/8] PasteLikeProxy -> PasteProxy --- restkit/contrib/wsgi_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index 3717932f..e24124d2 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -196,7 +196,7 @@ def make_host_proxy(global_config, uri=None, **local_config): 'upgrade', ) -class PasteLikeProxy(object): +class PasteProxy(object): def __init__(self, address, allowed_request_methods=(), suppress_http_headers=(), stream=False, **kwargs): From 5c34e7c859dcc7c617aa946432335b9cd718b68d Mon Sep 17 00:00:00 2001 From: Alan Justino Date: Mon, 29 Apr 2013 00:39:48 -0300 Subject: [PATCH 8/8] parse_headers -> filter_headers --- restkit/contrib/wsgi_proxy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/restkit/contrib/wsgi_proxy.py b/restkit/contrib/wsgi_proxy.py index e24124d2..7f921ecb 100644 --- a/restkit/contrib/wsgi_proxy.py +++ b/restkit/contrib/wsgi_proxy.py @@ -260,7 +260,7 @@ def __call__(self, environ, start_response): res = conn.request(u'%s://%s%s' % (self.scheme, self.host, path), environ['REQUEST_METHOD'], body=body, headers=headers) - headers_out = parse_headers(res.headerslist, stream=self.stream) + headers_out = filter_headers(res.headerslist, stream=self.stream) status = res.status start_response(status, headers_out) @@ -282,9 +282,9 @@ def __call__(self, environ, start_response): return body -def parse_headers(headers_list, stream=False): +def filter_headers(headers_list, stream=False): """ - Turn a Message object into a list of WSGI-style headers. + Filter headers list of WSGI-style headers. """ headers_out = [] for header, value in headers_list: