import pycurl from StringIO import StringIO import logging import os i

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import pycurl
from StringIO import StringIO
import logging
import os
import urllib
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
from proxy.models import Proxy
from proxy.const import *
from util.watcher import Watcher
# We should ignore SIGPIPE when using pycurl.NOSIGNAL - see
# the libcurl tutorial for more info.
try:
import signal
from signal import SIGPIPE, SIG_IGN
signal.signal(signal.SIGPIPE, signal.SIG_IGN)
except ImportError:
pass
def request(url, timeout=10, connect_timeout=5, proxy=None, proxy_type='http', post=None):
ios = StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.MAXREDIRS, 5)
curl.setopt(pycurl.CONNECTTIMEOUT, connect_timeout)
curl.setopt(pycurl.TIMEOUT, timeout)
curl.setopt(pycurl.NOSIGNAL, 1)
curl.setopt(pycurl.WRITEFUNCTION, ios.write)
if proxy:
curl.setopt(pycurl.PROXY, str(proxy))
# CURLPROXY_HTTP, CURLPROXY_HTTP_1_0 (added in 7.19.4),
# CURLPROXY_SOCKS4 (added in 7.15.2), CURLPROXY_SOCKS5,
# CURLPROXY_SOCKS4A (added in 7.18.0) and CURLPROXY_SOCKS5_HOSTNAME (added in 7.18.0)
ptype = getattr(pycurl, 'PROXYTYPE_%s' % proxy_type.upper())
curl.setopt(pycurl.PROXYTYPE, ptype)
if post:
post_data = urllib.urlencode(post)
curl.setopt(pycurl.POSTFIELDS, post_data)
curl.perform()
response = {'data': ios.getvalue(), 'time': curl.getinfo(pycurl.TOTAL_TIME)}
curl.close()
return response