#!/usr/bin/env python
#-*- encoding: UTF-8 -*-
class HTTP:
"""
Class-wrapper for standart library urllib and urllib2.
HTTP class instance have 3 default attributes.
ENCODING - encoding of page (by default utf-8).
TIMEOUT - timeot of getting pages (by default 20).
HEADERS - headers of request (by default opera 9.64 without zlib compression).
(c) login999
"""
def __init__(self):
"""
Initialization , no input data.
"""
import urllib2
from urllib import quote
from urllib import unquote
from urllib import urlencode
from urllib import quote_plus
from urllib import unquote_plus
from cookielib import CookieJar
self.__Cookies = CookieJar
self.__Urlencode = urlencode
self.__ProxyHandler = urllib2.ProxyHandler
self.__BuildOpener = urllib2.build_opener
self.__InstallOpener = urllib2.install_opener
self.__HTTPCookieProcessor = urllib2.HTTPCookieProcessor
self.__Open = urllib2.urlopen
self.__Request = urllib2.Request
self.ENCODING = "UTF-8"
self.TIMEOUT = 20
self.HEADERS = {
"User-Agent" : "Opera/9.64 (Windows NT 5.1; U; en) Presto/2.1.1",
"Accept" : "text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1",
"Accept-Language" : "ru,uk-UA;q=0.9,uk;q=0.8,en;q=0.7",
"Accept-Charset" : "iso-8859-1, utf-8, utf-16, *;q=0.1",
"Accept-Encoding" : "identity, *;q=0",
"Connection" : "Keep-Alive"
}
def make_urlencoded(self, data):
"""
Method for making data in "urlencoded" format.
Input - data as dict.
"""
for key, value in data.items():
if isinstance(key, unicode):
del data[key]
key = key.encode(self.ENCODING)
data[key] = value
if isinstance(value, unicode):
data[key] = value.encode(self.ENCODING)
return self.__Urlencode(data)
def make_opener(self, cookies, proxy):
"""
Method for creating an opener instance.
Input data - cookies as cookielib.CookieJar instance and proxy
as string in ip:port fromat (10.10.10.10:3128).
"""
if proxy:
proxy_handler = self.__ProxyHandler(
{
"http": "http://"+proxy+"/",
"https": "http://"+proxy+"/"
}
)
opener = self.__BuildOpener(self.__HTTPCookieProcessor(cookies), proxy_handler)
else:
opener = self.__BuildOpener(self.__HTTPCookieProcessor(cookies))
return opener
def GET(self, URI, cookies=None, proxy=None, headers=None, bytes=None, page_encoding=None, timeout=None, raw=False):
"""
Method for getting pages.
Input data: URI as string(URI of page), cookies as cookielib.CookieJar instance, proxy
as string in ip:port fromat, headers as dict, bytes as int(count of bytes which are must be read from page body)
page_encoding as string, timeout as int, raw as bool.
"""
if page_encoding is None:
page_encoding = self.ENCODING
if timeout is None:
timeout = self.TIMEOUT
if not cookies:
cookies = self.__Cookies()
if headers:
headers_temp = self.HEADERS
for header in headers:
headers_temp[header] = headers[header]
headers = headers_temp
elif not headers:
headers = self.HEADERS
response = {
"STATUS":"",
"COOKIES":"",
"BODY":"",
"PROXY":""
}
opener = self.make_opener(cookies, proxy)
self.__InstallOpener(opener)
if not bytes:
request = self.__Request(URI, None, headers)
try:
response["BODY"] = self.__Open(url=request, data=None, timeout=timeout).read()
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
else:
request = self.__Request(URI, None, headers)
try:
response["BODY"] = self.__Open(url=request, data=None, timeout=timeout).read(bytes)
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
response["COOKIES"] = cookies
response["PROXY"] = proxy
if not raw:
response["BODY"] = response["BODY"].decode(page_encoding, "replace")
return response
def POST(self, URI, data=None, cookies=None, proxy=None, headers=None, bytes=None, page_encoding=None, timeout=None, raw=False):
"""
Method for posting data on pages.
Input data: URI as string(URI of page), data as dict, cookies as cookielib.CookieJar instance, proxy
as string in ip:port fromat, headers as dict, bytes as int(count of bytes which are must be read from page body)
page_encoding as string, timeout as int, raw as bool.
"""
if page_encoding is None:
page_encoding = self.ENCODING
if not cookies:
cookies = self.__Cookies()
if timeout is None:
timeout = self.TIMEOUT
if isinstance(data, dict):
data = self.make_urlencoded(data)
if headers:
headers_temp = self.HEADERS
for header in headers:
headers_temp[header] = headers[header]
headers = headers_temp
elif not headers:
headers = self.HEADERS
response = {
"STATUS":"",
"COOKIES":"",
"BODY":"",
"PROXY":""
}
opener = self.make_opener(cookies, proxy)
self.__InstallOpener(opener)
if not bytes:
request = self.__Request(URI, None, headers)
try:
response["BODY"] = self.__Open(url=request, data=data, timeout=timeout).read()
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
else:
request = self.__Request(URI, None, headers)
try:
response["BODY"] = self.__Open(url=request, data=data, timeout=timeout).read(bytes)
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
response["COOKIES"] = cookies
response["PROXY"] = proxy
if not raw:
response["BODY"] = response["BODY"].decode(page_encoding, "replace")
return response
def GET_HEADERS(self, URI, data=None, cookies=None, proxy=None, headers=None, timeout=None):
"""
Method for getting headers of page.
Input data: URI as string(URI of page), data as dict, cookies as cookielib.CookieJar instance, proxy
as string in ip:port fromat, headers as dict, page_encoding as string, timeout as int
"""
if not cookies:
cookies = self.__Cookies()
if timeout is None:
timeout = self.TIMEOUT
if isinstance(data, dict):
data = self.make_urlencoded(data)
if headers:
headers_temp = self.HEADERS
for header in headers:
headers_temp[header] = headers[header]
headers = headers_temp
elif not headers:
headers = self.HEADERS
response = {
"STATUS":"",
"COOKIES":"",
"HEADERS":"",
"PROXY":""
}
opener = self.make_opener(cookies, proxy)
self.__InstallOpener(opener)
try:
request = self.__Request(URI, data, headers)
response["HEADERS"] = self.__Open(url=request, data=None, timeout=timeout).info()
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
response["COOKIES"] = cookies
response["PROXY"] = proxy
return response
def GET_CODE(self, URI, data=None, cookies=None, proxy=None, headers=None, timeout=None):
"""
Method for getting code of answer of page.
Input data: URI as string(URI of page), data as dict, cookies as cookielib.CookieJar instance, proxy
as string in ip:port fromat, headers as dict, page_encoding as string, timeout as int.
"""
if not cookies:
cookies = self.__Cookies()
if timeout is None:
timeout = self.TIMEOUT
if isinstance(data, dict):
data = self.make_urlencoded(data)
if headers:
headers_temp = self.HEADERS
for header in headers:
headers_temp[header] = headers[header]
headers = headers_temp
elif not headers:
headers = self.HEADERS
response = {
"STATUS":"",
"COOKIES":"",
"CODE":"",
"PROXY":""
}
opener = self.make_opener(cookies, proxy)
self.__InstallOpener(opener)
try:
request = self.__Request(URI, data, headers)
response["CODE"] = self.__Open(url=request, data=None, timeout=timeout).getcode()
response["STATUS"] = "OK"
except Exception, e:
response["STATUS"] = e
response["COOKIES"] = cookies
response["PROXY"] = proxy
return response
Шняга