Here are the examples of the python api urllib.request.HTTPCookieProcessor taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
18 Examples
3
Example 1
def get_response(url, faker = False):
# install cookies
if cookies:
opener = request.build_opener(request.HTTPCookieProcessor(cookies))
request.install_opener(opener)
if faker:
response = request.urlopen(request.Request(url, headers = fake_headers), None)
else:
response = request.urlopen(url)
data = response.read()
if response.info().get('Content-Encoding') == 'gzip':
data = ungzip(data)
elif response.info().get('Content-Encoding') == 'deflate':
data = undeflate(data)
response.data = data
return response
3
Example 2
Project: nsetools Source File: nse.py
def nse_opener(self):
"""
builds opener for urllib2
:return: opener object
"""
cj = CookieJar()
return build_opener(HTTPCookieProcessor(cj))
3
Example 3
def get_unread_count(self):
if not self.logged_in:
self.login()
try:
f = self.opener.open(self.bookmark_url)
root = ET.fromstring(f.read())
return int(root.attrib["newposts"]) - self.offset
except Exception:
self.cj.clear()
self.opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(self.cj))
self.logged_in = False
3
Example 4
Project: etlpy Source File: spider.py
def autologin(self, loginItem):
if loginItem.postdata is None:
return;
import http.cookiejar
cj = http.cookiejar.CookieJar()
pro = urllib.request.HTTPCookieProcessor(cj)
opener = urllib.request.build_opener(pro)
t = [(r, loginItem.Headers[r]) for r in loginItem.Headers];
opener.addheaders = t;
binary_data = loginItem.postdata.encode('utf-8')
op = opener.open(loginItem.Url, binary_data)
data = op.read().decode('utf-8')
print(data)
self.HttpItem.Url = op.url;
return opener;
3
Example 5
def __init__(self, debug=False, Server=None):
"""
Initialise the API connection, log in, and store authentication cookie
"""
self.set_config_data(Server)
self.urlOpener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor())
if debug: self.urlOpener.add_handler(urllib.request.HTTPHandler(debuglevel=1))
self.reqCount = 1
self._login() #login to the zenoss system
3
Example 6
def get_response(url, faker=False):
# install cookies
if cookies:
opener = request.build_opener(request.HTTPCookieProcessor(cookies))
request.install_opener(opener)
if faker:
response = request.urlopen(request.Request(url, headers=fake_headers), None)
else:
response = request.urlopen(url)
data = response.read()
if response.info().get('Content-Encoding') == 'gzip':
data = ungzip(data)
elif response.info().get('Content-Encoding') == 'deflate':
data = undeflate(data)
response.data = data
return response
3
Example 7
Project: SublimeIPythonNotebook Source File: ipy_connection.py
def install_proxy_opener():
global cookies
cookies=CookieJar()
proxy = ProxyHandler({})
opener = build_opener(proxy, HTTPCookieProcessor(cookies))
install_opener(opener)
3
Example 8
Project: dotnetpaddingoracle Source File: mtools.py
def defaultCreateOpener(withCookieJar = True, withBurpProxy = True):
global cookieJar
if withCookieJar:
cookieJar = urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())
proxy_handler = None
if withBurpProxy:
proxy_handler = urllib.request.ProxyHandler({'https': 'https://127.0.0.1:8080/', 'http': 'http://127.0.0.1:8080/'})
ret = None
if withCookieJar and withBurpProxy:
ret = urllib.request.build_opener(proxy_handler, SmartRedirectHandler(), cookieJar)
elif withCookieJar:
ret = urllib.request.build_opener(SmartRedirectHandler(), cookieJar)
elif withBurpProxy:
ret = urllib.request.build_opener(proxy_handler, SmartRedirectHandler())
return ret
3
Example 9
Project: odoorpc Source File: jsonrpclib.py
def __init__(self, host, port, timeout=120, ssl=False, opener=None):
self._root_url = "{http}{host}:{port}".format(
http=(ssl and "https://" or "http://"), host=host, port=port)
self._timeout = timeout
self._builder = URLBuilder(self)
self._opener = opener
if not opener:
cookie_jar = CookieJar()
self._opener = build_opener(HTTPCookieProcessor(cookie_jar))
3
Example 10
Project: odoorpc Source File: __init__.py
def __init__(self, host, port=8069, timeout=120, version=None,
deserialize=True):
super(ConnectorJSONRPC, self).__init__(host, port, timeout, version)
self.deserialize = deserialize
# One URL opener (with cookies handling) shared between
# JSON and HTTP requests
cookie_jar = CookieJar()
self._opener = build_opener(
HTTPCookieProcessor(cookie_jar))
self._proxy_json, self._proxy_http = self._get_proxies()
0
Example 11
def __init__(self):
self.DEBUG = False
self.appid = 'wx782c26e4c19acffb'
self.uuid = ''
self.base_uri = ''
self.redirect_uri = ''
self.uin = ''
self.sid = ''
self.skey = ''
self.pass_ticket = ''
self.deviceId = 'e' + repr(random.random())[2:17]
self.BaseRequest = {}
self.synckey = ''
self.SyncKey = []
self.User = []
self.MemberList = []
self.ContactList = []
self.GroupList = []
self.autoReplyMode = False
self.syncHost = ''
self._handlers = dict((k, []) for k in self.message_types)
self._handlers['location'] = []
self._handlers['all'] = []
self._filters = dict()
opener = request.build_opener(request.HTTPCookieProcessor(CookieJar()))
opener.addheaders = [('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'),
('Referer','https://wx2.qq.com/')]
request.install_opener(opener)
0
Example 12
def __init__(self, host, username, password, port=8443, version='v2', site_id='default'):
"""Create a Controller object.
Arguments:
host -- the address of the controller host; IP or name
username -- the username to log in with
password -- the password to log in with
port -- the port of the controller host
version -- the base version of the controller API [v2|v3|v4|v5]
site_id -- the site ID to connect to (UniFi >= 3.x)
"""
self.host = host
self.port = port
self.version = version
self.username = username
self.password = password
self.site_id = site_id
self.url = 'https://' + host + ':' + str(port) + '/'
self.api_url = self.url + self._construct_api_path(version)
log.debug('Controller for %s', self.url)
cj = cookielib.CookieJar()
if PYTHON_VERSION == 2:
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
elif PYTHON_VERSION == 3:
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
self._login(version)
0
Example 13
Project: i3pystatus Source File: modsde.py
def init(self):
self.cj = http.cookiejar.CookieJar()
self.opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(self.cj))
0
Example 14
Project: etlpy Source File: spider.py
def GetHTML(self, destUrl=None):
if destUrl is None:
destUrl = self.Url;
destUrl = self.PraseURL(destUrl);
socket.setdefaulttimeout(self.Timeout);
cj = http.cookiejar.CookieJar()
pro = urllib.request.HTTPCookieProcessor(cj)
opener = urllib.request.build_opener(pro)
t = [(r, self.Headers[r]) for r in self.Headers];
opener.addheaders = t;
binary_data = self.postdata.encode('utf-8')
try:
destUrl.encode('ascii')
except UnicodeEncodeError:
destUrl = iriToUri(destUrl)
try:
if self.postdata=='':
page=opener.open(destUrl);
else:
page = opener.open(destUrl, binary_data)
html = page.read()
except Exception as e:
print(e);
return ""
if page.info().get('Content-Encoding') == 'gzip':
html = gzip.decompress(html)
encoding = charset.search(str(html))
if encoding is not None:
encoding = encoding.group(1);
if encoding is None:
encoding = 'utf-8'
try:
html=html.decode(encoding)
except UnicodeDecodeError as e:
print(e);
import chardet
encoding= chardet.detect(html)
html=html.decode(encoding);
return html;
0
Example 15
Project: GetOldTweets-python Source File: TweetManager.py
@staticmethod
def getJsonReponse(tweetCriteria, refreshCursor, cookieJar):
url = "https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&%smax_position=%s"
urlGetData = ''
if hasattr(tweetCriteria, 'username'):
urlGetData += ' from:' + tweetCriteria.username
if hasattr(tweetCriteria, 'since'):
urlGetData += ' since:' + tweetCriteria.since
if hasattr(tweetCriteria, 'until'):
urlGetData += ' until:' + tweetCriteria.until
if hasattr(tweetCriteria, 'querySearch'):
urlGetData += ' ' + tweetCriteria.querySearch
if hasattr(tweetCriteria, 'lang'):
urlLang = 'lang=' + tweetCriteria.lang + '&'
else:
urlLang = ''
url = url % (urllib.parse.quote(urlGetData), urlLang, refreshCursor)
#print(url)
headers = [
('Host', "twitter.com"),
('User-Agent', "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"),
('Accept', "application/json, text/javascript, */*; q=0.01"),
('Accept-Language', "de,en-US;q=0.7,en;q=0.3"),
('X-Requested-With', "XMLHttpRequest"),
('Referer', url),
('Connection', "keep-alive")
]
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))
opener.addheaders = headers
try:
response = opener.open(url)
jsonResponse = response.read()
except:
#print("Twitter weird response. Try to see on browser: ", url)
print("Twitter weird response. Try to see on browser: https://twitter.com/search?q=%s&src=typd" % urllib.parse.quote(urlGetData))
print("Unexpected error:", sys.exc_info()[0])
sys.exit()
return
dataJson = json.loads(jsonResponse.decode())
return dataJson
0
Example 16
Project: alfred-python Source File: request.py
def __init__(self, url, **kwargs):
self.request = None
self.response = None
self.code = -1
self.info = {}
self.cookieJar = None
self.reason = ''
data = kwargs.get('data', None)
if data:
if isinstance(data, dict):
data = urlencode(data)
if not isinstance(data, basestring):
raise ValueError('data must be string or dict')
request_type = kwargs.get('type', 'POST')
if data and isinstance(request_type, basestring) and request_type.upper()!='POST':
url = '{}?{}'.format(url, data)
data = None # GET data must be None
self.request = urlRequest(url, data)
# referer
referer = kwargs.get('referer', None)
if referer:
self.request.add_header('referer', referer)
# user-agent
user_agent = kwargs.get('user_agent', None)
if user_agent:
self.request.add_header('User-Agent', user_agent)
# auth
auth = kwargs.get('auth', None)
if auth and isinstance(auth, dict) and 'usr' in auth:
auth_string = base64.b64encode('{}:{}'.format(auth.get('usr',''), auth.get('pwd','')))
self.request.add_header('Authorization', 'Basic {}'.format(auth_string))
# cookie
cookie = kwargs.get('cookie', None)
cj = None
if cookie:
if isinstance(cookie, CookieJar):
cj = cookie
elif isinstance(cookie, dict):
result = []
for k, v in cookie.items():
result.append('{}={}'.format(k, v))
cookie = '; '.join(result)
elif isinstance(cookie, Cookie.BaseCookie):
cookie = cookie.output(header='')
if isinstance(cookie, basestring):
self.request.add_header('Cookie', cookie)
if cj is None:
cj = CookieJar()
#! TODO: proxy
# build opener
debuglevel = 1 if kwargs.get('debug', False) else 0
opener = build_opener(
HTTPHandler(debuglevel=debuglevel),
HTTPSHandler(debuglevel=debuglevel),
HTTPCookieProcessor(cj)
)
# timeout
timeout = kwargs.get('timeout')
if not isinstance(timeout, int):
timeout = _DEFAULT_TIMEOUT
try:
self.response = opener.open(self.request, timeout=timeout)
self.code = self.response.getcode()
self.header = self.response.info().dict
self.cookieJar = cj
except HTTPError as e:
self.code = e.code
self.reason = '{}'.format(e)
raise e
except URLError as e:
self.code = -1
self.reason = e.reason
raise e
except Exception as e:
self.code = -1
self.reason = '{}'.format(e)
raise e
0
Example 17
def login(self, username, pwd, cookie_file):
""""
Login with use name, password and cookies.
(1) If cookie file exists then try to load cookies;
(2) If no cookies found then do login
"""
# If cookie file exists then try to load cookies
if os.path.exists(cookie_file):
try:
cookie_jar = cookielib.LWPCookieJar(cookie_file)
cookie_jar.load(ignore_discard=True, ignore_expires=True)
loaded = 1
except cookielib.LoadError:
loaded = 0
print('Loading cookies error')
#install loaded cookies for urllib2
if loaded:
cookie_support = urllib2.HTTPCookieProcessor(cookie_jar)
opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
urllib2.install_opener(opener)
print('Loading cookies success')
return 1
else:
return self.do_login(username, pwd, cookie_file)
else: #If no cookies found
return self.do_login(username, pwd, cookie_file)
0
Example 18
def do_login(self, username, pwd, cookie_file):
""""
Perform login action with use name, password and saving cookies.
@param username: login user name
@param pwd: login password
@param cookie_file: file name where to save cookies when login succeeded
"""
# POST data per LOGIN WEIBO, these fields can be captured using httpfox extension in FIrefox
login_data = {
'entry': 'weibo',
'gateway': '1',
'from': '',
'savestate': '7',
'userticket': '1',
'pagerefer': '',
'vsnf': '1',
'su': '',
'service': 'miniblog',
'servertime': '',
'nonce': '',
'pwencode': 'rsa2',
'rsakv': '',
'sp': '',
'encoding': 'UTF-8',
'prelt': '45',
'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
'returntype': 'META'
}
cookie_jar2 = cookielib.LWPCookieJar()
cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2)
opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler)
urllib2.install_opener(opener2)
login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)'
try:
servertime, nonce, rsakv = self.get_prelogin_status(username)
except:
return
#Fill POST data
print('starting to set login_data')
login_data['servertime'] = servertime
login_data['nonce'] = nonce
login_data['su'] = self.get_user(username)
login_data['sp'] = self.get_pwd_rsa(pwd, servertime, nonce)
login_data['rsakv'] = rsakv
login_data = urllib.urlencode(login_data)
http_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'}
req_login = urllib2.Request(
url=login_url,
data=login_data,
headers=http_headers
)
result = urllib2.urlopen(req_login)
text = result.read()
p = re.compile('location\.replace\(\'(.*?)\'\)')
#在使用httpfox登录调试时,我获取的返回参数 location.replace('http://weibo.com 这里使用的是单引号 原来的正则中匹配的是双引号# 导致没有login_url得到 单引号本身在re中无需转义
#p = re.compile('location\.replace\(\B'(.*?)'\B\)') 经调试 这样子是错误的 re中非的使用\'才能表达单引号
try:
#Search login redirection URL
login_url = p.search(text).group(1)
data = urllib2.urlopen(login_url).read()
#Verify login feedback, check whether result is TRUE
patt_feedback = 'feedBackUrlCallBack\((.*)\)'
p = re.compile(patt_feedback, re.MULTILINE)
feedback = p.search(data).group(1)
feedback_json = json.loads(feedback)
if feedback_json['result']:
cookie_jar2.save(cookie_file, ignore_discard=True, ignore_expires=True)
return 1
else:
return 0
except:
return 0