urllib.request.HTTPCookieProcessor

Here are the examples of the python api urllib.request.HTTPCookieProcessor taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

18 Examples 7

Example 1

Project: you-get Source File: common.py
Function: get_response
def get_response(url, faker = False):
    # install cookies
    if cookies:
        opener = request.build_opener(request.HTTPCookieProcessor(cookies))
        request.install_opener(opener)

    if faker:
        response = request.urlopen(request.Request(url, headers = fake_headers), None)
    else:
        response = request.urlopen(url)

    data = response.read()
    if response.info().get('Content-Encoding') == 'gzip':
        data = ungzip(data)
    elif response.info().get('Content-Encoding') == 'deflate':
        data = undeflate(data)
    response.data = data
    return response

Example 2

Project: nsetools Source File: nse.py
    def nse_opener(self):
        """
        builds opener for urllib2
        :return: opener object
        """
        cj = CookieJar()
        return build_opener(HTTPCookieProcessor(cj))

Example 3

Project: i3pystatus Source File: modsde.py
Function: get_unread_count
    def get_unread_count(self):
        if not self.logged_in:
            self.login()

        try:
            f = self.opener.open(self.bookmark_url)
            root = ET.fromstring(f.read())
            return int(root.attrib["newposts"]) - self.offset
        except Exception:
            self.cj.clear()
            self.opener = urllib.request.build_opener(
                urllib.request.HTTPCookieProcessor(self.cj))
            self.logged_in = False

Example 4

Project: etlpy Source File: spider.py
    def autologin(self, loginItem):
        if loginItem.postdata is None:
            return;
        import http.cookiejar
        cj = http.cookiejar.CookieJar()
        pro = urllib.request.HTTPCookieProcessor(cj)
        opener = urllib.request.build_opener(pro)
        t = [(r, loginItem.Headers[r]) for r in loginItem.Headers];
        opener.addheaders = t;
        binary_data = loginItem.postdata.encode('utf-8')
        op = opener.open(loginItem.Url, binary_data)
        data = op.read().decode('utf-8')
        print(data)
        self.HttpItem.Url = op.url;
        return opener;

Example 5

Project: Nagstamon Source File: zenoss_api.py
Function: init
    def __init__(self, debug=False, Server=None):
        """
        Initialise the API connection, log in, and store authentication cookie
        """

        self.set_config_data(Server)
        self.urlOpener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor())
        if debug: self.urlOpener.add_handler(urllib.request.HTTPHandler(debuglevel=1))
        self.reqCount = 1

        self._login() #login to the zenoss system

Example 6

Project: GUI-YouGet Source File: common.py
Function: get_response
def get_response(url, faker=False):
    # install cookies
    if cookies:
        opener = request.build_opener(request.HTTPCookieProcessor(cookies))
        request.install_opener(opener)

    if faker:
        response = request.urlopen(request.Request(url, headers=fake_headers), None)
    else:
        response = request.urlopen(url)

    data = response.read()
    if response.info().get('Content-Encoding') == 'gzip':
        data = ungzip(data)
    elif response.info().get('Content-Encoding') == 'deflate':
        data = undeflate(data)
    response.data = data
    return response

Example 7

Project: SublimeIPythonNotebook Source File: ipy_connection.py
def install_proxy_opener():
    global cookies
    cookies=CookieJar()
    proxy = ProxyHandler({})
    opener = build_opener(proxy, HTTPCookieProcessor(cookies))
    install_opener(opener)

Example 8

Project: dotnetpaddingoracle Source File: mtools.py
def defaultCreateOpener(withCookieJar = True, withBurpProxy = True):
	global cookieJar

	if withCookieJar:
		cookieJar = urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())

	proxy_handler = None	
	if withBurpProxy:
		proxy_handler = urllib.request.ProxyHandler({'https': 'https://127.0.0.1:8080/', 'http': 'http://127.0.0.1:8080/'})
	
	ret = None
	if withCookieJar and withBurpProxy:
		ret = urllib.request.build_opener(proxy_handler, SmartRedirectHandler(), cookieJar)
	elif withCookieJar:
		ret = urllib.request.build_opener(SmartRedirectHandler(), cookieJar)
	elif withBurpProxy:
		ret = urllib.request.build_opener(proxy_handler, SmartRedirectHandler())
	return ret

Example 9

Project: odoorpc Source File: jsonrpclib.py
    def __init__(self, host, port, timeout=120, ssl=False, opener=None):
        self._root_url = "{http}{host}:{port}".format(
            http=(ssl and "https://" or "http://"), host=host, port=port)
        self._timeout = timeout
        self._builder = URLBuilder(self)
        self._opener = opener
        if not opener:
            cookie_jar = CookieJar()
            self._opener = build_opener(HTTPCookieProcessor(cookie_jar))

Example 10

Project: odoorpc Source File: __init__.py
    def __init__(self, host, port=8069, timeout=120, version=None,
                 deserialize=True):
        super(ConnectorJSONRPC, self).__init__(host, port, timeout, version)
        self.deserialize = deserialize
        # One URL opener (with cookies handling) shared between
        # JSON and HTTP requests
        cookie_jar = CookieJar()
        self._opener = build_opener(
            HTTPCookieProcessor(cookie_jar))
        self._proxy_json, self._proxy_http = self._get_proxies()

Example 11

Project: WxRobot Source File: webwxapi.py
Function: init
    def __init__(self):
        self.DEBUG = False
        self.appid = 'wx782c26e4c19acffb'
        self.uuid = ''
        self.base_uri = ''
        self.redirect_uri = ''
        self.uin = ''
        self.sid = ''
        self.skey = ''
        self.pass_ticket = ''
        self.deviceId = 'e' + repr(random.random())[2:17]
        self.BaseRequest = {}
        self.synckey = ''
        self.SyncKey = []
        self.User = []
        self.MemberList = []
        self.ContactList = []
        self.GroupList = []
        self.autoReplyMode = False
        self.syncHost = ''

        self._handlers = dict((k, []) for k in self.message_types)
        self._handlers['location'] = []
        self._handlers['all'] = []

        self._filters = dict()

        opener = request.build_opener(request.HTTPCookieProcessor(CookieJar()))
        opener.addheaders = [('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'),
                             ('Referer','https://wx2.qq.com/')]
        request.install_opener(opener)

Example 12

Project: unifi-api Source File: controller.py
Function: init
    def __init__(self, host, username, password, port=8443, version='v2', site_id='default'):
        """Create a Controller object.

        Arguments:
            host     -- the address of the controller host; IP or name
            username -- the username to log in with
            password -- the password to log in with
            port     -- the port of the controller host
            version  -- the base version of the controller API [v2|v3|v4|v5]
            site_id  -- the site ID to connect to (UniFi >= 3.x)

        """

        self.host = host
        self.port = port
        self.version = version
        self.username = username
        self.password = password
        self.site_id = site_id
        self.url = 'https://' + host + ':' + str(port) + '/'
        self.api_url = self.url + self._construct_api_path(version)

        log.debug('Controller for %s', self.url)

        cj = cookielib.CookieJar()
        if PYTHON_VERSION == 2:
            self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        elif PYTHON_VERSION == 3:
            self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))

        self._login(version)

Example 13

Project: i3pystatus Source File: modsde.py
    def init(self):
        self.cj = http.cookiejar.CookieJar()
        self.opener = urllib.request.build_opener(
            urllib.request.HTTPCookieProcessor(self.cj))

Example 14

Project: etlpy Source File: spider.py
    def GetHTML(self, destUrl=None):
        if destUrl is None:
            destUrl = self.Url;
        destUrl = self.PraseURL(destUrl);
        socket.setdefaulttimeout(self.Timeout);
        cj = http.cookiejar.CookieJar()
        pro = urllib.request.HTTPCookieProcessor(cj)
        opener = urllib.request.build_opener(pro)
        t = [(r, self.Headers[r]) for r in self.Headers];
        opener.addheaders = t;
        binary_data = self.postdata.encode('utf-8')
        try:
            destUrl.encode('ascii')
        except UnicodeEncodeError:
            destUrl =  iriToUri(destUrl)

        try:
            if self.postdata=='':
                page=opener.open(destUrl);
            else:
                page = opener.open(destUrl, binary_data)
            html = page.read()
        except Exception as e:
            print(e);
            return ""


        if page.info().get('Content-Encoding') == 'gzip':
            html = gzip.decompress(html)
        encoding = charset.search(str(html))
        if encoding is not None:
            encoding = encoding.group(1);
        if encoding is None:
            encoding = 'utf-8'
        try:
            html=html.decode(encoding)
        except UnicodeDecodeError as e:
            print(e);
            import chardet
            encoding= chardet.detect(html)
            html=html.decode(encoding);

        return html;

Example 15

Project: GetOldTweets-python Source File: TweetManager.py
	@staticmethod
	def getJsonReponse(tweetCriteria, refreshCursor, cookieJar):
		url = "https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&%smax_position=%s"
		
		urlGetData = ''
		if hasattr(tweetCriteria, 'username'):
			urlGetData += ' from:' + tweetCriteria.username
			
		if hasattr(tweetCriteria, 'since'):
			urlGetData += ' since:' + tweetCriteria.since
			
		if hasattr(tweetCriteria, 'until'):
			urlGetData += ' until:' + tweetCriteria.until
			
		if hasattr(tweetCriteria, 'querySearch'):
			urlGetData += ' ' + tweetCriteria.querySearch
			
		if hasattr(tweetCriteria, 'lang'):
			urlLang = 'lang=' + tweetCriteria.lang + '&'
		else:
			urlLang = ''
		url = url % (urllib.parse.quote(urlGetData), urlLang, refreshCursor)
		#print(url)

		headers = [
			('Host', "twitter.com"),
			('User-Agent', "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"),
			('Accept', "application/json, text/javascript, */*; q=0.01"),
			('Accept-Language', "de,en-US;q=0.7,en;q=0.3"),
			('X-Requested-With', "XMLHttpRequest"),
			('Referer', url),
			('Connection', "keep-alive")
		]

		opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))
		opener.addheaders = headers

		try:
			response = opener.open(url)
			jsonResponse = response.read()
		except:
			#print("Twitter weird response. Try to see on browser: ", url)
			print("Twitter weird response. Try to see on browser: https://twitter.com/search?q=%s&src=typd" % urllib.parse.quote(urlGetData))
			print("Unexpected error:", sys.exc_info()[0])
			sys.exit()
			return
		
		dataJson = json.loads(jsonResponse.decode())
		
		return dataJson		

Example 16

Project: alfred-python Source File: request.py
    def __init__(self, url, **kwargs):
        self.request = None
        self.response = None
        self.code = -1
        self.info = {}
        self.cookieJar = None
        self.reason = ''

        data = kwargs.get('data', None)
        if data:
            if isinstance(data, dict):
                data = urlencode(data)
            if not isinstance(data, basestring):
                raise ValueError('data must be string or dict')

        request_type = kwargs.get('type', 'POST')
        if data and isinstance(request_type, basestring) and request_type.upper()!='POST':
            url = '{}?{}'.format(url, data)
            data = None # GET data must be None

        self.request = urlRequest(url, data)

        # referer
        referer = kwargs.get('referer', None)
        if referer:
            self.request.add_header('referer', referer)

        # user-agent
        user_agent = kwargs.get('user_agent', None)
        if user_agent:
            self.request.add_header('User-Agent', user_agent)

        # auth
        auth = kwargs.get('auth', None)
        if auth and isinstance(auth, dict) and 'usr' in auth:
            auth_string = base64.b64encode('{}:{}'.format(auth.get('usr',''), auth.get('pwd','')))
            self.request.add_header('Authorization', 'Basic {}'.format(auth_string))  

        # cookie
        cookie = kwargs.get('cookie', None)
        cj = None
        if cookie:
            if isinstance(cookie, CookieJar):
                cj = cookie
            elif isinstance(cookie, dict):
                result = []
                for k, v in cookie.items():
                    result.append('{}={}'.format(k, v))
                cookie = '; '.join(result)
            elif isinstance(cookie, Cookie.BaseCookie):
                cookie = cookie.output(header='')
            if isinstance(cookie, basestring):
                self.request.add_header('Cookie', cookie)
        if cj is None:
            cj = CookieJar()

        #! TODO: proxy


        # build opener
        debuglevel = 1 if kwargs.get('debug', False) else 0
        opener = build_opener(
            HTTPHandler(debuglevel=debuglevel),
            HTTPSHandler(debuglevel=debuglevel),
            HTTPCookieProcessor(cj)
        )

        # timeout
        timeout = kwargs.get('timeout')
        if not isinstance(timeout, int):
            timeout = _DEFAULT_TIMEOUT

        try:
            self.response = opener.open(self.request, timeout=timeout)
            self.code = self.response.getcode()
            self.header = self.response.info().dict
            self.cookieJar = cj
        except HTTPError as e:
            self.code = e.code
            self.reason = '{}'.format(e)
            raise e
        except URLError as e:
            self.code = -1
            self.reason = e.reason
            raise e
        except Exception as e:
            self.code = -1
            self.reason = '{}'.format(e)
            raise e

Example 17

Project: weibo Source File: Lweibo.py
Function: log_in
    def login(self, username, pwd, cookie_file):
        """"
            Login with use name, password and cookies.
            (1) If cookie file exists then try to load cookies;
            (2) If no cookies found then do login
        """
        # If cookie file exists then try to load cookies
        if os.path.exists(cookie_file):
            try:
                cookie_jar = cookielib.LWPCookieJar(cookie_file)
                cookie_jar.load(ignore_discard=True, ignore_expires=True)
                loaded = 1
            except cookielib.LoadError:
                loaded = 0
                print('Loading cookies error')

            #install loaded cookies for urllib2
            if loaded:
                cookie_support = urllib2.HTTPCookieProcessor(cookie_jar)
                opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
                urllib2.install_opener(opener)
                print('Loading cookies success')
                return 1
            else:
                return self.do_login(username, pwd, cookie_file)

        else:  #If no cookies found
            return self.do_login(username, pwd, cookie_file)

Example 18

Project: weibo Source File: Lweibo.py
Function: do_login
    def do_login(self, username, pwd, cookie_file):
        """"
        Perform login action with use name, password and saving cookies.
        @param username: login user name
        @param pwd: login password
        @param cookie_file: file name where to save cookies when login succeeded
        """
        # POST data per LOGIN WEIBO, these fields can be captured using httpfox extension in FIrefox
        login_data = {
            'entry': 'weibo',
            'gateway': '1',
            'from': '',
            'savestate': '7',
            'userticket': '1',
            'pagerefer': '',
            'vsnf': '1',
            'su': '',
            'service': 'miniblog',
            'servertime': '',
            'nonce': '',
            'pwencode': 'rsa2',
            'rsakv': '',
            'sp': '',
            'encoding': 'UTF-8',
            'prelt': '45',
            'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',
            'returntype': 'META'
        }

        cookie_jar2 = cookielib.LWPCookieJar()
        cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2)
        opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler)
        urllib2.install_opener(opener2)
        login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)'
        try:
            servertime, nonce, rsakv = self.get_prelogin_status(username)
        except:
            return

        #Fill POST data
        print('starting to set login_data')
        login_data['servertime'] = servertime
        login_data['nonce'] = nonce
        login_data['su'] = self.get_user(username)
        login_data['sp'] = self.get_pwd_rsa(pwd, servertime, nonce)
        login_data['rsakv'] = rsakv
        login_data = urllib.urlencode(login_data)
        http_headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:8.0) Gecko/20100101 Firefox/8.0'}
        req_login = urllib2.Request(
            url=login_url,
            data=login_data,
            headers=http_headers
        )
        result = urllib2.urlopen(req_login)
        text = result.read()
        p = re.compile('location\.replace\(\'(.*?)\'\)')
        #在使用httpfox登录调试时,我获取的返回参数  location.replace('http://weibo.com 这里使用的是单引号 原来的正则中匹配的是双引号# 导致没有login_url得到 单引号本身在re中无需转义
        #p = re.compile('location\.replace\(\B'(.*?)'\B\)') 经调试 这样子是错误的 re中非的使用\'才能表达单引号
        try:
            #Search login redirection URL
            login_url = p.search(text).group(1)
            data = urllib2.urlopen(login_url).read()
            #Verify login feedback, check whether result is TRUE
            patt_feedback = 'feedBackUrlCallBack\((.*)\)'
            p = re.compile(patt_feedback, re.MULTILINE)

            feedback = p.search(data).group(1)
            feedback_json = json.loads(feedback)
            if feedback_json['result']:
                cookie_jar2.save(cookie_file, ignore_discard=True, ignore_expires=True)
                return 1
            else:
                return 0
        except:
            return 0