My favorites | Sign in
Project Home
Repository:
Checkout   Browse   Changes   Clones  
Changes to /couchdb/http.py
e260990c44ff vs. b7ada64cd590 Compare: vs.  Format:
Revision b7ada64cd590
Go to: 
Project members, sign in to write a code review
/couchdb/http.py   e260990c44ff /couchdb/http.py   b7ada64cd590
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 # 3 #
4 # Copyright (C) 2009 Christopher Lenz 4 # Copyright (C) 2009 Christopher Lenz
5 # All rights reserved. 5 # All rights reserved.
6 # 6 #
7 # This software is licensed as described in the file COPYING, which 7 # This software is licensed as described in the file COPYING, which
8 # you should have received as part of this distribution. 8 # you should have received as part of this distribution.
9 9
10 """Simple HTTP client implementation based on the ``httplib`` module in the 10 """Simple HTTP client implementation based on the ``httplib`` module in the
11 standard library. 11 standard library.
12 """ 12 """
13 13
14 from base64 import b64encode 14 from base64 import b64encode
15 from datetime import datetime 15 from datetime import datetime
16 import errno 16 import errno
17 from httplib import BadStatusLine, HTTPConnection, HTTPSConnection 17 from httplib import BadStatusLine, HTTPConnection, HTTPSConnection
18 import socket 18 import socket
19 import time 19 import time
20 try: 20 try:
21 from cStringIO import StringIO 21 from cStringIO import StringIO
22 except ImportError: 22 except ImportError:
23 from StringIO import StringIO 23 from StringIO import StringIO
24 import sys 24 import sys
25 try: 25 try:
26 from threading import Lock 26 from threading import Lock
27 except ImportError: 27 except ImportError:
28 from dummy_threading import Lock 28 from dummy_threading import Lock
29 import urllib 29 import urllib
30 from urlparse import urlsplit, urlunsplit 30 from urlparse import urlsplit, urlunsplit
31 from email.Utils import parsedate
31 32
32 from couchdb import json 33 from couchdb import json
33 34
34 __all__ = ['HTTPError', 'PreconditionFailed', 'ResourceNotFound', 35 __all__ = ['HTTPError', 'PreconditionFailed', 'ResourceNotFound',
35 'ResourceConflict', 'ServerError', 'Unauthorized', 'RedirectLimit', 36 'ResourceConflict', 'ServerError', 'Unauthorized', 'RedirectLimit',
36 'Session', 'Resource'] 37 'Session', 'Resource']
37 __docformat__ = 'restructuredtext en' 38 __docformat__ = 'restructuredtext en'
38 39
39 40
40 if sys.version < '2.6': 41 if sys.version < '2.6':
41 42
42 class TimeoutMixin: 43 class TimeoutMixin:
43 """Helper mixin to add timeout before socket connection""" 44 """Helper mixin to add timeout before socket connection"""
44 45
45 # taken from original python2.5 httplib source code with timeout setting added 46 # taken from original python2.5 httplib source code with timeout setting added
46 def connect(self): 47 def connect(self):
47 """Connect to the host and port specified in __init__.""" 48 """Connect to the host and port specified in __init__."""
48 msg = "getaddrinfo returns an empty list" 49 msg = "getaddrinfo returns an empty list"
49 for res in socket.getaddrinfo(self.host, self.port, 0, 50 for res in socket.getaddrinfo(self.host, self.port, 0,
50 socket.SOCK_STREAM): 51 socket.SOCK_STREAM):
51 af, socktype, proto, canonname, sa = res 52 af, socktype, proto, canonname, sa = res
52 try: 53 try:
53 self.sock = socket.socket(af, socktype, proto) 54 self.sock = socket.socket(af, socktype, proto)
54 if self.debuglevel > 0: 55 if self.debuglevel > 0:
55 print "connect: (%s, %s)" % (self.host, self.port) 56 print "connect: (%s, %s)" % (self.host, self.port)
56 57
57 # setting socket timeout 58 # setting socket timeout
58 self.sock.settimeout(self.timeout) 59 self.sock.settimeout(self.timeout)
59 60
60 self.sock.connect(sa) 61 self.sock.connect(sa)
61 except socket.error, msg: 62 except socket.error, msg:
62 if self.debuglevel > 0: 63 if self.debuglevel > 0:
63 print 'connect fail:', (self.host, self.port) 64 print 'connect fail:', (self.host, self.port)
64 if self.sock: 65 if self.sock:
65 self.sock.close() 66 self.sock.close()
66 self.sock = None 67 self.sock = None
67 continue 68 continue
68 break 69 break
69 if not self.sock: 70 if not self.sock:
70 raise socket.error, msg 71 raise socket.error, msg
71 72
72 _HTTPConnection = HTTPConnection 73 _HTTPConnection = HTTPConnection
73 _HTTPSConnection = HTTPSConnection 74 _HTTPSConnection = HTTPSConnection
74 75
75 class HTTPConnection(TimeoutMixin, _HTTPConnection): 76 class HTTPConnection(TimeoutMixin, _HTTPConnection):
76 def __init__(self, *a, **k): 77 def __init__(self, *a, **k):
77 timeout = k.pop('timeout', None) 78 timeout = k.pop('timeout', None)
78 _HTTPConnection.__init__(self, *a, **k) 79 _HTTPConnection.__init__(self, *a, **k)
79 self.timeout = timeout 80 self.timeout = timeout
80 81
81 class HTTPSConnection(TimeoutMixin, _HTTPSConnection): 82 class HTTPSConnection(TimeoutMixin, _HTTPSConnection):
82 def __init__(self, *a, **k): 83 def __init__(self, *a, **k):
83 timeout = k.pop('timeout', None) 84 timeout = k.pop('timeout', None)
84 _HTTPSConnection.__init__(self, *a, **k) 85 _HTTPSConnection.__init__(self, *a, **k)
85 self.timeout = timeout 86 self.timeout = timeout
86 87
87 88
88 if sys.version < '2.7': 89 if sys.version < '2.7':
89 90
90 from httplib import CannotSendHeader, _CS_REQ_STARTED, _CS_REQ_SENT 91 from httplib import CannotSendHeader, _CS_REQ_STARTED, _CS_REQ_SENT
91 92
92 class NagleMixin: 93 class NagleMixin:
93 """ 94 """
94 Mixin to upgrade httplib connection types so headers and body can be 95 Mixin to upgrade httplib connection types so headers and body can be
95 sent at the same time to avoid triggering Nagle's algorithm. 96 sent at the same time to avoid triggering Nagle's algorithm.
96 97
97 Based on code originally copied from Python 2.7's httplib module. 98 Based on code originally copied from Python 2.7's httplib module.
98 """ 99 """
99 100
100 def endheaders(self, message_body=None): 101 def endheaders(self, message_body=None):
101 if self.__dict__['_HTTPConnection__state'] == _CS_REQ_STARTED: 102 if self.__dict__['_HTTPConnection__state'] == _CS_REQ_STARTED:
102 self.__dict__['_HTTPConnection__state'] = _CS_REQ_SENT 103 self.__dict__['_HTTPConnection__state'] = _CS_REQ_SENT
103 else: 104 else:
104 raise CannotSendHeader() 105 raise CannotSendHeader()
105 self._send_output(message_body) 106 self._send_output(message_body)
106 107
107 def _send_output(self, message_body=None): 108 def _send_output(self, message_body=None):
108 self._buffer.extend(("", "")) 109 self._buffer.extend(("", ""))
109 msg = "\r\n".join(self._buffer) 110 msg = "\r\n".join(self._buffer)
110 del self._buffer[:] 111 del self._buffer[:]
111 if isinstance(message_body, str): 112 if isinstance(message_body, str):
112 msg += message_body 113 msg += message_body
113 message_body = None 114 message_body = None
114 self.send(msg) 115 self.send(msg)
115 if message_body is not None: 116 if message_body is not None:
116 self.send(message_body) 117 self.send(message_body)
117 118
118 class HTTPConnection(NagleMixin, HTTPConnection): 119 class HTTPConnection(NagleMixin, HTTPConnection):
119 pass 120 pass
120 121
121 class HTTPSConnection(NagleMixin, HTTPSConnection): 122 class HTTPSConnection(NagleMixin, HTTPSConnection):
122 pass 123 pass
123 124
124 125
125 class HTTPError(Exception): 126 class HTTPError(Exception):
126 """Base class for errors based on HTTP status codes >= 400.""" 127 """Base class for errors based on HTTP status codes >= 400."""
127 128
128 129
129 class PreconditionFailed(HTTPError): 130 class PreconditionFailed(HTTPError):
130 """Exception raised when a 412 HTTP error is received in response to a 131 """Exception raised when a 412 HTTP error is received in response to a
131 request. 132 request.
132 """ 133 """
133 134
134 135
135 class ResourceNotFound(HTTPError): 136 class ResourceNotFound(HTTPError):
136 """Exception raised when a 404 HTTP error is received in response to a 137 """Exception raised when a 404 HTTP error is received in response to a
137 request. 138 request.
138 """ 139 """
139 140
140 141
141 class ResourceConflict(HTTPError): 142 class ResourceConflict(HTTPError):
142 """Exception raised when a 409 HTTP error is received in response to a 143 """Exception raised when a 409 HTTP error is received in response to a
143 request. 144 request.
144 """ 145 """
145 146
146 147
147 class ServerError(HTTPError): 148 class ServerError(HTTPError):
148 """Exception raised when an unexpected HTTP error is received in response 149 """Exception raised when an unexpected HTTP error is received in response
149 to a request. 150 to a request.
150 """ 151 """
151 152
152 153
153 class Unauthorized(HTTPError): 154 class Unauthorized(HTTPError):
154 """Exception raised when the server requires authentication credentials 155 """Exception raised when the server requires authentication credentials
155 but either none are provided, or they are incorrect. 156 but either none are provided, or they are incorrect.
156 """ 157 """
157 158
158 159
159 class RedirectLimit(Exception): 160 class RedirectLimit(Exception):
160 """Exception raised when a request is redirected more often than allowed 161 """Exception raised when a request is redirected more often than allowed
161 by the maximum number of redirections. 162 by the maximum number of redirections.
162 """ 163 """
163 164
164 165
165 CHUNK_SIZE = 1024 * 8 166 CHUNK_SIZE = 1024 * 8
166 167
167 def cache_sort(i): 168 def cache_sort(i):
168 t = time.mktime(time.strptime(i[1][1]['Date'][5:-4], '%d %b %Y %H:%M:%S')) 169 return datetime.fromtimestamp(time.mktime(parsedate(i[1][1]['Date'])))
169 return datetime.fromtimestamp(t)
170 170
171 class ResponseBody(object): 171 class ResponseBody(object):
172 172
173 def __init__(self, resp, callback): 173 def __init__(self, resp, callback):
174 self.resp = resp 174 self.resp = resp
175 self.callback = callback 175 self.callback = callback
176 176
177 def read(self, size=None): 177 def read(self, size=None):
178 bytes = self.resp.read(size) 178 bytes = self.resp.read(size)
179 if size is None or len(bytes) < size: 179 if size is None or len(bytes) < size:
180 self.close() 180 self.close()
181 return bytes 181 return bytes
182 182
183 def close(self): 183 def close(self):
184 while not self.resp.isclosed(): 184 while not self.resp.isclosed():
185 self.resp.read(CHUNK_SIZE) 185 self.resp.read(CHUNK_SIZE)
186 if self.callback: 186 if self.callback:
187 self.callback() 187 self.callback()
188 self.callback = None 188 self.callback = None
189 189
190 def iterchunks(self): 190 def iterchunks(self):
191 assert self.resp.msg.get('transfer-encoding') == 'chunked' 191 assert self.resp.msg.get('transfer-encoding') == 'chunked'
192 while True: 192 while True:
193 if self.resp.isclosed(): 193 if self.resp.isclosed():
194 break 194 break
195 chunksz = int(self.resp.fp.readline().strip(), 16) 195 chunksz = int(self.resp.fp.readline().strip(), 16)
196 if not chunksz: 196 if not chunksz:
197 self.resp.fp.read(2) #crlf 197 self.resp.fp.read(2) #crlf
198 self.resp.close() 198 self.resp.close()
199 self.callback() 199 self.callback()
200 break 200 break
201 chunk = self.resp.fp.read(chunksz) 201 chunk = self.resp.fp.read(chunksz)
202 for ln in chunk.splitlines(): 202 for ln in chunk.splitlines():
203 yield ln 203 yield ln
204 self.resp.fp.read(2) #crlf 204 self.resp.fp.read(2) #crlf
205 205
206 206
207 RETRYABLE_ERRORS = frozenset([ 207 RETRYABLE_ERRORS = frozenset([
208 errno.EPIPE, errno.ETIMEDOUT, 208 errno.EPIPE, errno.ETIMEDOUT,
209 errno.ECONNRESET, errno.ECONNREFUSED, errno.ECONNABORTED, 209 errno.ECONNRESET, errno.ECONNREFUSED, errno.ECONNABORTED,
210 errno.EHOSTDOWN, errno.EHOSTUNREACH, 210 errno.EHOSTDOWN, errno.EHOSTUNREACH,
211 errno.ENETRESET, errno.ENETUNREACH, errno.ENETDOWN 211 errno.ENETRESET, errno.ENETUNREACH, errno.ENETDOWN
212 ]) 212 ])
213 213
214 214
215 class Session(object): 215 class Session(object):
216 216
217 def __init__(self, cache=None, timeout=None, max_redirects=5, 217 def __init__(self, cache=None, timeout=None, max_redirects=5,
218 retry_delays=[0], retryable_errors=RETRYABLE_ERRORS): 218 retry_delays=[0], retryable_errors=RETRYABLE_ERRORS):
219 """Initialize an HTTP client session. 219 """Initialize an HTTP client session.
220 220
221 :param cache: an instance with a dict-like interface or None to allow 221 :param cache: an instance with a dict-like interface or None to allow
222 Session to create a dict for caching. 222 Session to create a dict for caching.
223 :param timeout: socket timeout in number of seconds, or `None` for no 223 :param timeout: socket timeout in number of seconds, or `None` for no
224 timeout (the default) 224 timeout (the default)
225 :param retry_delays: list of request retry delays. 225 :param retry_delays: list of request retry delays.
226 """ 226 """
227 from couchdb import __version__ as VERSION 227 from couchdb import __version__ as VERSION
228 self.user_agent = 'CouchDB-Python/%s' % VERSION 228 self.user_agent = 'CouchDB-Python/%s' % VERSION
229 # XXX We accept a `cache` dict arg, but the ref gets overwritten later 229 # XXX We accept a `cache` dict arg, but the ref gets overwritten later
230 # during cache cleanup. Do we remove the cache arg (does using a shared 230 # during cache cleanup. Do we remove the cache arg (does using a shared
231 # Session instance cover the same use cases?) or fix the cache cleanup? 231 # Session instance cover the same use cases?) or fix the cache cleanup?
232 # For now, let's just assign the dict to the Cache instance to retain 232 # For now, let's just assign the dict to the Cache instance to retain
233 # current behaviour. 233 # current behaviour.
234 if cache is not None: 234 if cache is not None:
235 cache_by_url = cache 235 cache_by_url = cache
236 cache = Cache() 236 cache = Cache()
237 cache.by_url = cache_by_url 237 cache.by_url = cache_by_url
238 else: 238 else:
239 cache = Cache() 239 cache = Cache()
240 self.cache = cache 240 self.cache = cache
241 self.max_redirects = max_redirects 241 self.max_redirects = max_redirects
242 self.perm_redirects = {} 242 self.perm_redirects = {}
243 self.connection_pool = ConnectionPool(timeout) 243 self.connection_pool = ConnectionPool(timeout)
244 self.retry_delays = list(retry_delays) # We don't want this changing on us. 244 self.retry_delays = list(retry_delays) # We don't want this changing on us.
245 self.retryable_errors = set(retryable_errors) 245 self.retryable_errors = set(retryable_errors)
246 246
247 def request(self, method, url, body=None, headers=None, credentials=None, 247 def request(self, method, url, body=None, headers=None, credentials=None,
248 num_redirects=0): 248 num_redirects=0):
249 if url in self.perm_redirects: 249 if url in self.perm_redirects:
250 url = self.perm_redirects[url] 250 url = self.perm_redirects[url]
251 method = method.upper() 251 method = method.upper()
252 252
253 if headers is None: 253 if headers is None:
254 headers = {} 254 headers = {}
255 headers.setdefault('Accept', 'application/json') 255 headers.setdefault('Accept', 'application/json')
256 headers['User-Agent'] = self.user_agent 256 headers['User-Agent'] = self.user_agent
257 257
258 cached_resp = None 258 cached_resp = None
259 if method in ('GET', 'HEAD'): 259 if method in ('GET', 'HEAD'):
260 cached_resp = self.cache.get(url) 260 cached_resp = self.cache.get(url)
261 if cached_resp is not None: 261 if cached_resp is not None:
262 etag = cached_resp[1].get('etag') 262 etag = cached_resp[1].get('etag')
263 if etag: 263 if etag:
264 headers['If-None-Match'] = etag 264 headers['If-None-Match'] = etag
265 265
266 if (body is not None and not isinstance(body, basestring) and 266 if (body is not None and not isinstance(body, basestring) and
267 not hasattr(body, 'read')): 267 not hasattr(body, 'read')):
268 body = json.encode(body).encode('utf-8') 268 body = json.encode(body).encode('utf-8')
269 headers.setdefault('Content-Type', 'application/json') 269 headers.setdefault('Content-Type', 'application/json')
270 270
271 if body is None: 271 if body is None:
272 headers.setdefault('Content-Length', '0') 272 headers.setdefault('Content-Length', '0')
273 elif isinstance(body, basestring): 273 elif isinstance(body, basestring):
274 headers.setdefault('Content-Length', str(len(body))) 274 headers.setdefault('Content-Length', str(len(body)))
275 else: 275 else:
276 headers['Transfer-Encoding'] = 'chunked' 276 headers['Transfer-Encoding'] = 'chunked'
277 277
278 authorization = basic_auth(credentials) 278 authorization = basic_auth(credentials)
279 if authorization: 279 if authorization:
280 headers['Authorization'] = authorization 280 headers['Authorization'] = authorization
281 281
282 path_query = urlunsplit(('', '') + urlsplit(url)[2:4] + ('',)) 282 path_query = urlunsplit(('', '') + urlsplit(url)[2:4] + ('',))
283 conn = self.connection_pool.get(url) 283 conn = self.connection_pool.get(url)
284 284
285 def _try_request_with_retries(retries): 285 def _try_request_with_retries(retries):
286 while True: 286 while True:
287 try: 287 try:
288 return _try_request() 288 return _try_request()
289 except socket.error, e: 289 except socket.error, e:
290 ecode = e.args[0] 290 ecode = e.args[0]
291 if ecode not in self.retryable_errors: 291 if ecode not in self.retryable_errors:
292 raise 292 raise
293 try: 293 try:
294 delay = retries.next() 294 delay = retries.next()
295 except StopIteration: 295 except StopIteration:
296 # No more retries, raise last socket error. 296 # No more retries, raise last socket error.
297 raise e 297 raise e
298 time.sleep(delay) 298 time.sleep(delay)
299 conn.close() 299 conn.close()
300 300
301 def _try_request(): 301 def _try_request():
302 try: 302 try:
303 conn.putrequest(method, path_query, skip_accept_encoding=True) 303 conn.putrequest(method, path_query, skip_accept_encoding=True)
304 for header in headers: 304 for header in headers:
305 conn.putheader(header, headers[header]) 305 conn.putheader(header, headers[header])
306 if body is None: 306 if body is None:
307 conn.endheaders() 307 conn.endheaders()
308 else: 308 else:
309 if isinstance(body, str): 309 if isinstance(body, str):
310 conn.endheaders(body) 310 conn.endheaders(body)
311 else: # assume a file-like object and send in chunks 311 else: # assume a file-like object and send in chunks
312 conn.endheaders() 312 conn.endheaders()
313 while 1: 313 while 1:
314 chunk = body.read(CHUNK_SIZE) 314 chunk = body.read(CHUNK_SIZE)
315 if not chunk: 315 if not chunk:
316 break 316 break
317 conn.send(('%x\r\n' % len(chunk)) + chunk + '\r\n') 317 conn.send(('%x\r\n' % len(chunk)) + chunk + '\r\n')
318 conn.send('0\r\n\r\n') 318 conn.send('0\r\n\r\n')
319 return conn.getresponse() 319 return conn.getresponse()
320 except BadStatusLine, e: 320 except BadStatusLine, e:
321 # httplib raises a BadStatusLine when it cannot read the status 321 # httplib raises a BadStatusLine when it cannot read the status
322 # line saying, "Presumably, the server closed the connection 322 # line saying, "Presumably, the server closed the connection
323 # before sending a valid response." 323 # before sending a valid response."
324 # Raise as ECONNRESET to simplify retry logic. 324 # Raise as ECONNRESET to simplify retry logic.
325 if e.line == '' or e.line == "''": 325 if e.line == '' or e.line == "''":
326 raise socket.error(errno.ECONNRESET) 326 raise socket.error(errno.ECONNRESET)
327 else: 327 else:
328 raise 328 raise
329 329
330 resp = _try_request_with_retries(iter(self.retry_delays)) 330 resp = _try_request_with_retries(iter(self.retry_delays))
331 status = resp.status 331 status = resp.status
332 332
333 # Handle conditional response 333 # Handle conditional response
334 if status == 304 and method in ('GET', 'HEAD'): 334 if status == 304 and method in ('GET', 'HEAD'):
335 resp.read() 335 resp.read()
336 self.connection_pool.release(url, conn) 336 self.connection_pool.release(url, conn)
337 status, msg, data = cached_resp 337 status, msg, data = cached_resp
338 if data is not None: 338 if data is not None:
339 data = StringIO(data) 339 data = StringIO(data)
340 return status, msg, data 340 return status, msg, data
341 elif cached_resp: 341 elif cached_resp:
342 self.cache.remove(url) 342 self.cache.remove(url)
343 343
344 # Handle redirects 344 # Handle redirects
345 if status == 303 or \ 345 if status == 303 or \
346 method in ('GET', 'HEAD') and status in (301, 302, 307): 346 method in ('GET', 'HEAD') and status in (301, 302, 307):
347 resp.read() 347 resp.read()
348 self.connection_pool.release(url, conn) 348 self.connection_pool.release(url, conn)
349 if num_redirects > self.max_redirects: 349 if num_redirects > self.max_redirects:
350 raise RedirectLimit('Redirection limit exceeded') 350 raise RedirectLimit('Redirection limit exceeded')
351 location = resp.getheader('location') 351 location = resp.getheader('location')
352 if status == 301: 352 if status == 301:
353 self.perm_redirects[url] = location 353 self.perm_redirects[url] = location
354 elif status == 303: 354 elif status == 303:
355 method = 'GET' 355 method = 'GET'
356 return self.request(method, location, body, headers, 356 return self.request(method, location, body, headers,
357 num_redirects=num_redirects + 1) 357 num_redirects=num_redirects + 1)
358 358
359 data = None 359 data = None
360 streamed = False 360 streamed = False
361 361
362 # Read the full response for empty responses so that the connection is 362 # Read the full response for empty responses so that the connection is
363 # in good state for the next request 363 # in good state for the next request
364 if method == 'HEAD' or resp.getheader('content-length') == '0' or \ 364 if method == 'HEAD' or resp.getheader('content-length') == '0' or \
365 status < 200 or status in (204, 304): 365 status < 200 or status in (204, 304):
366 resp.read() 366 resp.read()
367 self.connection_pool.release(url, conn) 367 self.connection_pool.release(url, conn)
368 368
369 # Buffer small non-JSON response bodies 369 # Buffer small non-JSON response bodies
370 elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE: 370 elif int(resp.getheader('content-length', sys.maxint)) < CHUNK_SIZE:
371 data = resp.read() 371 data = resp.read()
372 self.connection_pool.release(url, conn) 372 self.connection_pool.release(url, conn)
373 373
374 # For large or chunked response bodies, do not buffer the full body, 374 # For large or chunked response bodies, do not buffer the full body,
375 # and instead return a minimal file-like object 375 # and instead return a minimal file-like object
376 else: 376 else:
377 data = ResponseBody(resp, 377 data = ResponseBody(resp,
378 lambda: self.connection_pool.release(url, conn)) 378 lambda: self.connection_pool.release(url, conn))
379 streamed = True 379 streamed = True
380 380
381 # Handle errors 381 # Handle errors
382 if status >= 400: 382 if status >= 400:
383 ctype = resp.getheader('content-type') 383 ctype = resp.getheader('content-type')
384 if data is not None and 'application/json' in ctype: 384 if data is not None and 'application/json' in ctype:
385 data = json.decode(data) 385 data = json.decode(data)
386 error = data.get('error'), data.get('reason') 386 error = data.get('error'), data.get('reason')
387 elif method != 'HEAD': 387 elif method != 'HEAD':
388 error = resp.read() 388 error = resp.read()
389 self.connection_pool.release(url, conn) 389 self.connection_pool.release(url, conn)
390 else: 390 else:
391 error = '' 391 error = ''
392 if status == 401: 392 if status == 401:
393 raise Unauthorized(error) 393 raise Unauthorized(error)
394 elif status == 404: 394 elif status == 404:
395 raise ResourceNotFound(error) 395 raise ResourceNotFound(error)
396 elif status == 409: 396 elif status == 409:
397 raise ResourceConflict(error) 397 raise ResourceConflict(error)
398 elif status == 412: 398 elif status == 412:
399 raise PreconditionFailed(error) 399 raise PreconditionFailed(error)
400 else: 400 else:
401 raise ServerError((status, error)) 401 raise ServerError((status, error))
402 402
403 # Store cachable responses 403 # Store cachable responses
404 if not streamed and method == 'GET' and 'etag' in resp.msg: 404 if not streamed and method == 'GET' and 'etag' in resp.msg:
405 self.cache.put(url, (status, resp.msg, data)) 405 self.cache.put(url, (status, resp.msg, data))
406 406
407 if not streamed and data is not None: 407 if not streamed and data is not None:
408 data = StringIO(data) 408 data = StringIO(data)
409 409
410 return status, resp.msg, data 410 return status, resp.msg, data
411 411
412 412
413 class Cache(object): 413 class Cache(object):
414 """Content cache.""" 414 """Content cache."""
415 415
416 # Some random values to limit memory use 416 # Some random values to limit memory use
417 keep_size, max_size = 10, 75 417 keep_size, max_size = 10, 75
418 418
419 def __init__(self): 419 def __init__(self):
420 self.by_url = {} 420 self.by_url = {}
421 421
422 def get(self, url): 422 def get(self, url):
423 return self.by_url.get(url) 423 return self.by_url.get(url)
424 424
425 def put(self, url, response): 425 def put(self, url, response):
426 self.by_url[url] = response 426 self.by_url[url] = response
427 if len(self.by_url) > self.max_size: 427 if len(self.by_url) > self.max_size:
428 self._clean() 428 self._clean()
429 429
430 def remove(self, url): 430 def remove(self, url):
431 self.by_url.pop(url, None) 431 self.by_url.pop(url, None)
432 432
433 def _clean(self): 433 def _clean(self):
434 ls = sorted(self.by_url.iteritems(), key=cache_sort) 434 ls = sorted(self.by_url.iteritems(), key=cache_sort)
435 self.by_url = dict(ls[-self.keep_size:]) 435 self.by_url = dict(ls[-self.keep_size:])
436 436
437 437
438 class ConnectionPool(object): 438 class ConnectionPool(object):
439 """HTTP connection pool.""" 439 """HTTP connection pool."""
440 440
441 def __init__(self, timeout): 441 def __init__(self, timeout):
442 self.timeout = timeout 442 self.timeout = timeout
443 self.conns = {} # HTTP connections keyed by (scheme, host) 443 self.conns = {} # HTTP connections keyed by (scheme, host)
444 self.lock = Lock() 444 self.lock = Lock()
445 445
446 def get(self, url): 446 def get(self, url):
447 447
448 scheme, host = urlsplit(url, 'http', False)[:2] 448 scheme, host = urlsplit(url, 'http', False)[:2]
449 449
450 # Try to reuse an existing connection. 450 # Try to reuse an existing connection.
451 self.lock.acquire() 451 self.lock.acquire()
452 try: 452 try:
453 conns = self.conns.setdefault((scheme, host), []) 453 conns = self.conns.setdefault((scheme, host), [])
454 if conns: 454 if conns:
455 conn = conns.pop(-1) 455 conn = conns.pop(-1)
456 else: 456 else:
457 conn = None 457 conn = None
458 finally: 458 finally:
459 self.lock.release() 459 self.lock.release()
460 460
461 # Create a new connection if nothing was available. 461 # Create a new connection if nothing was available.
462 if conn is None: 462 if conn is None:
463 if scheme == 'http': 463 if scheme == 'http':
464 cls = HTTPConnection 464 cls = HTTPConnection
465 elif scheme == 'https': 465 elif scheme == 'https':
466 cls = HTTPSConnection 466 cls = HTTPSConnection
467 else: 467 else:
468 raise ValueError('%s is not a supported scheme' % scheme) 468 raise ValueError('%s is not a supported scheme' % scheme)
469 conn = cls(host, timeout=self.timeout) 469 conn = cls(host, timeout=self.timeout)
470 conn.connect() 470 conn.connect()
471 471
472 return conn 472 return conn
473 473
474 def release(self, url, conn): 474 def release(self, url, conn):
475 scheme, host = urlsplit(url, 'http', False)[:2] 475 scheme, host = urlsplit(url, 'http', False)[:2]
476 self.lock.acquire() 476 self.lock.acquire()
477 try: 477 try:
478 self.conns.setdefault((scheme, host), []).append(conn) 478 self.conns.setdefault((scheme, host), []).append(conn)
479 finally: 479 finally:
480 self.lock.release() 480 self.lock.release()
481 481
482 482
483 class Resource(object): 483 class Resource(object):
484 484
485 def __init__(self, url, session, headers=None): 485 def __init__(self, url, session, headers=None):
486 self.url, self.credentials = extract_credentials(url) 486 self.url, self.credentials = extract_credentials(url)
487 if session is None: 487 if session is None:
488 session = Session() 488 session = Session()
489 self.session = session 489 self.session = session
490 self.headers = headers or {} 490 self.headers = headers or {}
491 491
492 def __call__(self, *path): 492 def __call__(self, *path):
493 obj = type(self)(urljoin(self.url, *path), self.session) 493 obj = type(self)(urljoin(self.url, *path), self.session)
494 obj.credentials = self.credentials 494 obj.credentials = self.credentials
495 obj.headers = self.headers.copy() 495 obj.headers = self.headers.copy()
496 return obj 496 return obj
497 497
498 def delete(self, path=None, headers=None, **params): 498 def delete(self, path=None, headers=None, **params):
499 return self._request('DELETE', path, headers=headers, **params) 499 return self._request('DELETE', path, headers=headers, **params)
500 500
501 def get(self, path=None, headers=None, **params): 501 def get(self, path=None, headers=None, **params):
502 return self._request('GET', path, headers=headers, **params) 502 return self._request('GET', path, headers=headers, **params)
503 503
504 def head(self, path=None, headers=None, **params): 504 def head(self, path=None, headers=None, **params):
505 return self._request('HEAD', path, headers=headers, **params) 505 return self._request('HEAD', path, headers=headers, **params)
506 506
507 def post(self, path=None, body=None, headers=None, **params): 507 def post(self, path=None, body=None, headers=None, **params):
508 return self._request('POST', path, body=body, headers=headers, 508 return self._request('POST', path, body=body, headers=headers,
509 **params) 509 **params)
510 510
511 def put(self, path=None, body=None, headers=None, **params): 511 def put(self, path=None, body=None, headers=None, **params):
512 return self._request('PUT', path, body=body, headers=headers, **params) 512 return self._request('PUT', path, body=body, headers=headers, **params)
513 513
514 def delete_json(self, path=None, headers=None, **params): 514 def delete_json(self, path=None, headers=None, **params):
515 return self._request_json('DELETE', path, headers=headers, **params) 515 return self._request_json('DELETE', path, headers=headers, **params)
516 516
517 def get_json(self, path=None, headers=None, **params): 517 def get_json(self, path=None, headers=None, **params):
518 return self._request_json('GET', path, headers=headers, **params) 518 return self._request_json('GET', path, headers=headers, **params)
519 519
520 def post_json(self, path=None, body=None, headers=None, **params): 520 def post_json(self, path=None, body=None, headers=None, **params):
521 return self._request_json('POST', path, body=body, headers=headers, 521 return self._request_json('POST', path, body=body, headers=headers,
522 **params) 522 **params)
523 523
524 def put_json(self, path=None, body=None, headers=None, **params): 524 def put_json(self, path=None, body=None, headers=None, **params):
525 return self._request_json('PUT', path, body=body, headers=headers, 525 return self._request_json('PUT', path, body=body, headers=headers,
526 **params) 526 **params)
527 527
528 def _request(self, method, path=None, body=None, headers=None, **params): 528 def _request(self, method, path=None, body=None, headers=None, **params):
529 all_headers = self.headers.copy() 529 all_headers = self.headers.copy()
530 all_headers.update(headers or {}) 530 all_headers.update(headers or {})
531 if path is not None: 531 if path is not None:
532 url = urljoin(self.url, path, **params) 532 url = urljoin(self.url, path, **params)
533 else: 533 else:
534 url = urljoin(self.url, **params) 534 url = urljoin(self.url, **params)
535 return self.session.request(method, url, body=body, 535 return self.session.request(method, url, body=body,
536 headers=all_headers, 536 headers=all_headers,
537 credentials=self.credentials) 537 credentials=self.credentials)
538 538
539 def _request_json(self, method, path=None, body=None, headers=None, **params): 539 def _request_json(self, method, path=None, body=None, headers=None, **params):
540 status, headers, data = self._request(method, path, body=body, 540 status, headers, data = self._request(method, path, body=body,
541 headers=headers, **params) 541 headers=headers, **params)
542 if 'application/json' in headers.get('content-type'): 542 if 'application/json' in headers.get('content-type'):
543 data = json.decode(data.read()) 543 data = json.decode(data.read())
544 return status, headers, data 544 return status, headers, data
545 545
546 546
547 547
548 def extract_credentials(url): 548 def extract_credentials(url):
549 """Extract authentication (user name and password) credentials from the 549 """Extract authentication (user name and password) credentials from the
550 given URL. 550 given URL.
551 551
552 >>> extract_credentials('http://localhost:5984/_config/') 552 >>> extract_credentials('http://localhost:5984/_config/')
553 ('http://localhost:5984/_config/', None) 553 ('http://localhost:5984/_config/', None)
554 >>> extract_credentials('http://joe:secret@localhost:5984/_config/') 554 >>> extract_credentials('http://joe:secret@localhost:5984/_config/')
555 ('http://localhost:5984/_config/', ('joe', 'secret')) 555 ('http://localhost:5984/_config/', ('joe', 'secret'))
556 >>> extract_credentials('http://joe%40example.com:secret@localhost:5984/_config/') 556 >>> extract_credentials('http://joe%40example.com:secret@localhost:5984/_config/')
557 ('http://localhost:5984/_config/', ('joe@example.com', 'secret')) 557 ('http://localhost:5984/_config/', ('joe@example.com', 'secret'))
558 """ 558 """
559 parts = urlsplit(url) 559 parts = urlsplit(url)
560 netloc = parts[1] 560 netloc = parts[1]
561 if '@' in netloc: 561 if '@' in netloc:
562 creds, netloc = netloc.split('@') 562 creds, netloc = netloc.split('@')
563 credentials = tuple(urllib.unquote(i) for i in creds.split(':')) 563 credentials = tuple(urllib.unquote(i) for i in creds.split(':'))
564 parts = list(parts) 564 parts = list(parts)
565 parts[1] = netloc 565 parts[1] = netloc
566 else: 566 else:
567 credentials = None 567 credentials = None
568 return urlunsplit(parts), credentials 568 return urlunsplit(parts), credentials
569 569
570 570
571 def basic_auth(credentials): 571 def basic_auth(credentials):
572 if credentials: 572 if credentials:
573 return 'Basic %s' % b64encode('%s:%s' % credentials) 573 return 'Basic %s' % b64encode('%s:%s' % credentials)
574 574
575 575
576 def quote(string, safe=''): 576 def quote(string, safe=''):
577 if isinstance(string, unicode): 577 if isinstance(string, unicode):
578 string = string.encode('utf-8') 578 string = string.encode('utf-8')
579 return urllib.quote(string, safe) 579 return urllib.quote(string, safe)
580 580
581 581
582 def urlencode(data): 582 def urlencode(data):
583 if isinstance(data, dict): 583 if isinstance(data, dict):
584 data = data.items() 584 data = data.items()
585 params = [] 585 params = []
586 for name, value in data: 586 for name, value in data:
587 if isinstance(value, unicode): 587 if isinstance(value, unicode):
588 value = value.encode('utf-8') 588 value = value.encode('utf-8')
589 params.append((name, value)) 589 params.append((name, value))
590 return urllib.urlencode(params) 590 return urllib.urlencode(params)
591 591
592 592
593 def urljoin(base, *path, **query): 593 def urljoin(base, *path, **query):
594 """Assemble a uri based on a base, any number of path segments, and query 594 """Assemble a uri based on a base, any number of path segments, and query
595 string parameters. 595 string parameters.
596 596
597 >>> urljoin('http://example.org', '_all_dbs') 597 >>> urljoin('http://example.org', '_all_dbs')
598 'http://example.org/_all_dbs' 598 'http://example.org/_all_dbs'
599 599
600 A trailing slash on the uri base is handled gracefully: 600 A trailing slash on the uri base is handled gracefully:
601 601
602 >>> urljoin('http://example.org/', '_all_dbs') 602 >>> urljoin('http://example.org/', '_all_dbs')
603 'http://example.org/_all_dbs' 603 'http://example.org/_all_dbs'
604 604
605 And multiple positional arguments become path parts: 605 And multiple positional arguments become path parts:
606 606
607 >>> urljoin('http://example.org/', 'foo', 'bar') 607 >>> urljoin('http://example.org/', 'foo', 'bar')
608 'http://example.org/foo/bar' 608 'http://example.org/foo/bar'
609 609
610 All slashes within a path part are escaped: 610 All slashes within a path part are escaped:
611 611
612 >>> urljoin('http://example.org/', 'foo/bar') 612 >>> urljoin('http://example.org/', 'foo/bar')
613 'http://example.org/foo%2Fbar' 613 'http://example.org/foo%2Fbar'
614 >>> urljoin('http://example.org/', 'foo', '/bar/') 614 >>> urljoin('http://example.org/', 'foo', '/bar/')
615 'http://example.org/foo/%2Fbar%2F' 615 'http://example.org/foo/%2Fbar%2F'
616 616
617 >>> urljoin('http://example.org/', None) #doctest:+IGNORE_EXCEPTION_DETAIL 617 >>> urljoin('http://example.org/', None) #doctest:+IGNORE_EXCEPTION_DETAIL
618 Traceback (most recent call last): 618 Traceback (most recent call last):
619 ... 619 ...
620 TypeError: argument 2 to map() must support iteration 620 TypeError: argument 2 to map() must support iteration
621 """ 621 """
622 if base and base.endswith('/'): 622 if base and base.endswith('/'):
623 base = base[:-1] 623 base = base[:-1]
624 retval = [base] 624 retval = [base]
625 625
626 # build the path 626 # build the path
627 path = '/'.join([''] + [quote(s) for s in path]) 627 path = '/'.join([''] + [quote(s) for s in path])
628 if path: 628 if path:
629 retval.append(path) 629 retval.append(path)
630 630
631 # build the query string 631 # build the query string
632 params = [] 632 params = []
633 for name, value in query.items(): 633 for name, value in query.items():
634 if type(value) in (list, tuple): 634 if type(value) in (list, tuple):
635 params.extend([(name, i) for i in value if i is not None]) 635 params.extend([(name, i) for i in value if i is not None])
636 elif value is not None: 636 elif value is not None:
637 if value is True: 637 if value is True:
638 value = 'true' 638 value = 'true'
639 elif value is False: 639 elif value is False:
640 value = 'false' 640 value = 'false'
641 params.append((name, value)) 641 params.append((name, value))
642 if params: 642 if params:
643 retval.extend(['?', urlencode(params)]) 643 retval.extend(['?', urlencode(params)])
644 644
645 return ''.join(retval) 645 return ''.join(retval)
646 646
Powered by Google Project Hosting