Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use proxy_handler method in devtools-proxy

Best Python code snippet using devtools-proxy_python

xicidai_spider.py

Source:xicidai_spider.py

1#coding:utf82import urllib2,urllib3import cookielib4import re5import random6# è®¾ç½®æ¯å¦å¼å¯ä»£çæ±   0: è±é±ä»£ç  1 ä½¿ç¨åè´¹ä»£ç  2 ä¸å®ç¨ä»£ç7proxy_switch = 18max_retry = 59# åå»ºcookiehandler10cookie = cookielib.CookieJar()11cookie_handler = urllib2.HTTPCookieProcessor(cookie)12# åå»ºä»£çhandler13# proxy_handler = urllib2.ProxyHandler({'http' : '120.27.125.149:16816'})14def getProxyHandler(proxy_list):15    if proxy_switch == 0:16        proxy_handler = urllib2.ProxyHandler(17            {18                'http' : 'http://1752570559:wd0p04kd@120.27.125.149:16816',19                'https' : 'http://1752570559:wd0p04kd@120.27.125.149:16816'20            }21        ) # ä½¿ç¨ç§å¯ä»£ç22    elif proxy_switch == 1:23        # åè´¹ä»£çï¼ä»ç¬åçä»£çæ± ä¸è¯»å24        print 'ç¨åè´¹ä»£ç'25        # with open('proxy.txt','r') as f:26        #27        #     proxy_list = f.readlines()  #ä»æä»¶ä¸è¯»åçä»£çåè¡¨28        #     print proxy_list29        #     proxy_poll = []  # ä»£çæ± 30        #     for proxy in proxy_list:31        #         res = proxy.split(' ')32        #         proxy_poll.append( #åä»£çæ± ä¸å å¥ä¸ä¸ªä»£ç33        #             {34        #                 'http' : res[2].lower() + '://'  + res[0] + ':' + res[1],35        #                 'https' : res[2].lower() + '://'  + res[0] + ':' + res[1],36        #             }37        #         )38        print proxy_list39        proxy_poll = []  # ä»£çæ± 40        for proxy in proxy_list:41            res = proxy.split(' ')42            proxy_poll.append( #åä»£çæ± ä¸å å¥ä¸ä¸ªä»£ç43                {44                    'http' : res[2].lower() + '://'  + res[0] + ':' + res[1],45                    'https' : res[2].lower() + '://'  + res[0] + ':' + res[1],46                }47            )48        if proxy_poll:  #å¦ææä»£çåéæº ï¼å¦åä¸å®ç¨ä»£ç49            proxy = random.choice(proxy_poll)50        else:51            proxy = {}52        print proxy53        proxy_handler = urllib2.ProxyHandler(proxy)54    else:55        proxy_handler = urllib2.ProxyHandler()  # ä½¿ç¨ç§å¯ä»£ç56    return proxy_handler57def getOpener(proxy_handler):58    opener = urllib2.build_opener(proxy_handler)59    # urllib2.install_opener(opener)  # urllib2.urlopenæ¶åé½ç¨è¿ä¸ªopenerå»åè¯·æ±60    urllib2.install_opener(opener)61    opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36')]62    return opener63def getPage():64    base_url = 'http://www.xicidaili.com/nn/'65    # base_url = 'http://www.baidu.com/s?wd=ip'66    # response = opener.open(base_url)67    # print response.read()68    # exit()69    start = raw_input('èµ·å§ï¼')70    end = raw_input('ç»æï¼')71    with open('proxy.txt','r') as f:72        proxy_list = f.readlines()  # æå¼æ¶åä¸å®ä¸è½åæ¸ç©º73    with open('proxy.txt', 'w+') as f:74        for i in range(int(start), int(end)):75            fullurl = base_url + str(i)76            for i in range(max_retry): #å¾ªç¯å»å°è¯open77                try:78                    proxy_handler = getProxyHandler(proxy_list)79                    opener = getOpener(proxy_handler)80                    response = opener.open(fullurl,timeout=3)81                    break82                except (urllib2.URLError,Exception) as e:83                    response = None84                    print str(e)85            if response is not None:  # å¤æ¬¡å°è¯æåä»¥åå°±è¿è¡htmlè¯»å86                html = response.read()87            else:88                print 'ä»£çä¸å¯ç¨ï¼è¯·æ£æ¥ï¼ï¼'89                exit()90            # print html91            tr_pattern = re.compile(r'<tr.*?>.*?' + r'(?:<td.*?>(.*?)</td>.*?)' * 10 + r'.*?</tr>' ,re.S)92            tr_list = tr_pattern.findall(html)93            speed_pattern = re.compile(r'title="(.*?)"')94            for tr in tr_list:95                # è¿æ»¤ååºå°äº2ç§ç96                res = speed_pattern.search(tr[6])97                speed = res.group(1)98                speed = speed.replace('ç§','')99                # åæ´»æå¤§äº1å¤©ä»¥ä¸ç100                if float(speed) < 2 and 'å¤©' in tr[8]:101                    print 'crawling : %s' % tr[1]102                    f.write("%s %s %s %s %s\n" % (tr[1],tr[2],tr[5],speed,tr[8],))103if __name__ == '__main__':104    getPage()...

compat.py

Source:compat.py

1# -*- coding: utf-8 -*-2# flake8: noqa3"""Python 2/3 compatibility support."""4import sys5PY2 = sys.version_info[0] == 26PY3 = sys.version_info[0] == 37PY33 = sys.version_info[0:2] >= (3, 3)8if PY2:9	reload(sys)10	sys.setdefaultencoding('utf8')11	import urllib212	from urllib import urlencode13	from urllib2 import URLError14	from urllib2 import quote15	from urllib2 import unquote16	from urllib2 import urlopen17	from urlparse import parse_qsl18	from HTMLParser import HTMLParser19	def install_proxy(proxy_handler):20		"""21		install global proxy.22		:param proxy_handler:23			:samp:`{"http":"http://my.proxy.com:1234", "https":"https://my.proxy.com:1234"}`24		:return:25		"""26		proxy_support = urllib2.ProxyHandler(proxy_handler)27		opener = urllib2.build_opener(proxy_support)28		urllib2.install_opener(opener)29	def unescape(s):30		"""Strip HTML entries from a string."""31		html_parser = HTMLParser()32		return html_parser.unescape(s)33	def unicode(s):34		"""Encode a string to utf-8."""35		return s.encode('utf-8')36elif PY3:37	from urllib.error import URLError38	from urllib.parse import parse_qsl39	from urllib.parse import quote40	from urllib.parse import unquote41	from urllib.parse import urlencode42	from urllib.request import urlopen43	from urllib import request44	def install_proxy(proxy_handler):45		proxy_support = request.ProxyHandler(proxy_handler)46		opener = request.build_opener(proxy_support)47		request.install_opener(opener)48	def unicode(s):49		"""No-op."""50		return s51	if PY33:52		from html.parser import HTMLParser53		def unescape(s):54			"""Strip HTML entries from a string."""55			html_parser = HTMLParser()56			return html_parser.unescape(s)57	else:...

test_proxy.py

Source:test_proxy.py

1#coding=utf-82import urllib23#ä¸ä½¿ç¨ä»£ç4import urllib25response = urllib2.urlopen('http://101.201.175.89/dist/#/login')6html = response.read()7print("#ä¸ä½¿ç¨ä»£ç")8print(html)9#ä½¿ç¨ä»£çè®¿é®10proxy_handler = urllib2.ProxyHandler({'http': '124.232.163.39:3128'})11opener = urllib2.build_opener(proxy_handler)12r = opener.open('http://101.201.175.89/dist/#/login')13print("#ä½¿ç¨ä»£ç")14print(r.read())15# read16# urllib2æ¯Pythonæ ååºï¼åè½å¾å¼ºå¤§ï¼åªæ¯ä½¿ç¨èµ·æ¥ç¨å¾®éº»ç¦ä¸ç¹ãå¨Python 3ä¸ï¼urllib2ä¸åä¿çï¼è¿ç§»å°äºurllibæ¨¡åä¸ãurllib2ä¸éè¿ProxyHandleræ¥è®¾ç½®ä½¿ç¨ä»£çæå¡å¨ã17#18# proxy_handler = urllib2.ProxyHandler({'http': '121.193.143.249:80'})19# opener = urllib2.build_opener(proxy_handler)20# r = opener.open('http://httpbin.org/ip')21# print(r.read())22# ä¹å¯ä»¥ç¨install_openerå°éç½®å¥½çopenerå®è£å°å¨å±ç¯å¢ä¸ï¼è¿æ ·ææçurllib2.urlopené½ä¼èªå¨ä½¿ç¨ä»£çã23#24# urllib2.install_opener(opener)25# r = urllib2.urlopen('http://httpbin.org/ip')26# print(r.read())27# å¨Python 3ä¸ï¼ä½¿ç¨urllibã28#29# proxy_handler = urllib.request.ProxyHandler({'http': 'http://121.193.143.249:80/'})30# opener = urllib.request.build_opener(proxy_handler)31# r = opener.open('http://httpbin.org/ip')...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.