Класс позволяющий без лишнего гемороя работать с HTTP

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
'''
connection.py
Simple HTTP connection module, allowing access to resource through a sequence
of chained methods, ending in .execute() and .read().
2009, Arseniy Ivanov
Released under DWTHYW v. 1.0 license.
'''
import cookielib
import gzip
import httplib
import StringIO
import urllib
import urllib2
class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
def http_error_301(self, req, fp, code, msg, headers):
result = urllib2.HTTPRedirectHandler.http_error_301(
self, req, fp, code, msg, headers)
result.status = code
return result
def http_error_302(self, req, fp, code, msg, headers):
result = urllib2.HTTPRedirectHandler.http_error_302(
self, req, fp, code, msg, headers)
result.status = code
return result
class DefaultErrorHandler(urllib2.HTTPDefaultErrorHandler):
def http_error_default(self, req, fp, code, msg, headers):
result = urllib2.HTTPError(
req.get_full_url(), code, msg, headers, fp)
result.status = code
return result
class Conn:
'''Class Conn implments a simple connection class, using urllib2 and
httplib2, featuring cookielib support and gzipped content decoding. '''
__reqest = None
__opener = None
__fh = None
__host = ''
__page = ''
__extra_headers = {}
__data = {'get': {}, 'post': {}}
__cj = None
__result = {}
def __init__(self, host, page):
'''Construct a new Conn class.
Arguments:
host --- web host to connect to
page --- web host resource to fetch
'''
self.__host = host
self.__page = page
self.__data = {'get': {}, 'post': {}} # Reset request data
# Add class-wide headers to this instance
if not self.__class__.__extra_headers == {}:
self.__extra_headers = self.__class__.__extra_headers.copy()
return None
def reset_data(self):
'''Reset request data.
Returns instance.
'''
self.__data = {'get': {}, 'post': {}}
return self
def get_data(self, data):
'''Set GET data for the request.
Arguments:
data --- a dictionary of key-value pairs.
Throws TypeError if argument data is not a dictionary.
Returns instance.
'''
if not isinstance(data, dict):
raise TypeError, "Argument data is expected to be a dictionary."
self.__set_data('get', data)
return self
def post_data(self, data):
'''Set request POST data.
Arguments:
data --- a dictionary of key-value pairs.
Throws TypeError if argument data is not a dictionary.
Returns instance.
'''
if not isinstance(data, dict):
raise TypeError, "Argument data is expected to be a dictionary."
self.__set_data('post', data)
return self
def __set_data(self, method, data):
'''Private method to set passed data to for request method method and
data data.
Arguments:
method --- a valid HTTP/1.1 request method to have this data sent
via. Currently, POST or GET.
data --- a dictionary of key-value pairs to send.
Throws TypeError if the data argument is not a dictionary.
'''
if not isinstance(data, dict):
raise TypeError, "Argument data is expected to be a dictionary."
self.__data[method].update(data)
return None
def set_cookiejar(self, jar=None, default=True):
'''Set a cookielib.CookieJar to use on this request. Sets current
instance's cookie jar to jar, given one; sets cookie jar to class-wide
jar, if no arguments were given; disables use of cookie jar if
default is specified to be False.
Arguments:
jar --- cookielib.CookieJar object to use.
default --- use the class-wide CookieJar.
Throws TypeError if the jar argument is not None and not a
cookielib.CookieJar instance.
Returns instance.
'''
if not (jar or isinstance(jar, cookielib.CookieJar)):
raise TypeError, "Argument jar is expected to be cookielib.CookieJar instance."
# Jar is an instance of cookielib.CookieJar
if jar:
self.__cj = jar
# Using default cookie jar
elif not jar and default:
if not self.__class__.__cj == None:
self.__cj = self.__class__.__cj
else:
self.__cj = self.__class__.__cj = cookielib.CookieJar()
# Disable cookies
else:
self.__cj = None
return self
def __set_global_headers(self, type, content):
self.__class__.__extra_headers[type] = content
return None
def __set_extra_headers(self, type, content):
self.__extra_headers[type] = content
return None
def set_useragent(self, agentstring):
'''Set a custom user-agent field. Sets a class-wide User-Agent header.
Arguments:
agentstring --- user agent string.
Returns instance.
'''
self.__set_global_headers('User-Agent', agentstring)
return self
def set_referer(self, referer):
'''Set refering url.
Arguments:
referer --- refering page string, usually a URL.
Returns instance.
'''
self.__set_extra_headers('Referer', referer)
return self
def execute(self):
'''Execute request. Assembles the full url, using urllib.urlencode to
incorporate GET data into it, then generates a request object, sets
extra headers, adds POST data, if any. Finally, creates an opener and
sends the requet.
Uses SmartRedirectHandler class to handle the redirects.
Arguments:
None
Retruns instance.
'''
# Assemble request url
full_url = self.get_url()
# Create request
self.__request = urllib2.Request(full_url)
if len(self.__extra_headers) > 0:
for (k, v) in self.__extra_headers.items():
self.__request.add_header(k, v)
self.__request.add_header('Accept-encoding', 'gzip')
self.__request.add_header('Connection', 'close')
if not self.__data['post'] == {}:
self.__request.add_data(urllib.urlencode(self.__data['post']))
self.__opener = urllib2.build_opener(urllib2.HTTPHandler(), \
SmartRedirectHandler(), \
DefaultErrorHandler())
if not self.__cj == None:
self.__opener.add_handler(urllib2.HTTPCookieProcessor(self.__cj))
self.__fh = self.__opener.open(self.__request)
return self
def read(self):
'''Read the response to the request and return a dictionary of status,
request URL, and response body, where possible. Body will be
automatically decompressed from gzip, if content-encoding states gzip.
Returns HTTP status code and empty body, if the request has failed.
Arguments:
None
Returns response data dictionary.
'''
# Read the response
self.__result['data'] = self.__fh.read()
if hasattr(self.__fh, 'headers'):
# Handle compression
if self.__fh.headers.get('content-encoding') == 'gzip':
# data came back gzip-compressed, decompress it
self.__result['data'] = gzip.GzipFile(fileobj=StringIO.StringIO(self.__result['data'])).read()
# Handle redirection
if self.__fh.headers.get('location'):
self.__result['location'] = self.__fh.headers.get('location')
if hasattr(self.__fh, 'url'):
self.__result['url'] = self.__fh.url
self.__result['status'] = 200
if hasattr(self.__fh, 'status'):
self.__result['status'] = self.__fh.status
self.__fh.close()
return self.__result
def get_url(self):
'''Get the full (host/resourse?get-data) url of this request.
Returns string, url.
'''
url = '%s/%s?%s' % (self.__host, self.__page, urllib.urlencode(self.__data['get']))
return url