How to use parse_result method in Playwright Python
Best Python code snippet using playwright-python
parse_chengji_fc.py
Source:parse_chengji_fc.py 
1from lxml import etree2from crawler.collection.key_map import go007_zf_map3import re4import urllib.parse as urllib_parse5from crawler.parse.parse_base import ParseBase6from env.exception import ParseFailureException7class ChengJiFangChan(ParseBase):8 def __init__(self):9 pass10 def parse_zf_link(self, resp):11 e = etree.HTML(resp)12 hrefs = e.xpath('.//ul[@class="box"]/li/a[contains(@href,"htm")]/@href | \13 .//ul[@class="box"]/li/h2/a[contains(@href,"htm")]/@href')14 return hrefs15 def parse_zf_detail_html_a(self, res):16 17 parse_result_a = {18 'parse_type': 'A',19 'viewad_header': {20 'view_title': None,21 'view_actions': None,22 },23 'img_url': None,24 'viewad_box': {25 'msg_type': None,26 'source': None,27 'area': None,28 'price': None,29 'name': None,30 'phone': None,31 'address': None32 },33 'detail_view': {34 'detail_produce': {35 'decoration_situation': None,36 'storey': None,37 'deposit': None,38 'layout': None,39 'orientation': None,40 'housing_estate': None,41 },42 'view_box': None,43 'img_box': []44 }45 }46 e = etree.HTML(res)47 parse_status = {'status':-1}48 try:49 detail_top = e.xpath('.//div[@class="content"]/div[@class="left"]/div[@class="details_top"]')50 if detail_top:51 detail_top = detail_top[0]52 viewad_header = detail_top.xpath('./div[@class="viewad-header"]')53 pic_box = detail_top.xpath('./div[@class="pic_box"]')54 viewad_box = detail_top.xpath('./div[@class="viewad_box"]')55 detail_view = detail_top.xpath('./dl[@class="detail_view"]')56 # print(viewad_header, pic_box, viewad_box, detail_view)57 if viewad_header:58 viewad_header = viewad_header[0]59 view_title = viewad_header.xpath('./div[@class="viewad-title "]/h1/text()')60 view_actions = viewad_header.xpath('./div[@class="viewad-actions"]/span[1]/text()')61 if view_title:62 view_title = view_title[0].replace('\r\n', '').replace(' ', '')63 parse_result_a['viewad_header']['view_title'] = view_title64 # print(parse_result_a['viewad_header']['view_title'])65 if view_actions:66 parse_result_a['viewad_header']['view_actions'] = view_actions[0]67 # print(view_title, view_actions[0])68 if pic_box:69 pic_box = pic_box[0]70 # print('''pic_box''')71 img_url = pic_box.xpath('./img/@src')72 if img_url:73 parse_result_a['img_url'] = img_url[0]74 # print(img_url[0])75 if viewad_box:76 viewad_box = viewad_box[0]77 viewad_meta = viewad_box.xpath('.//ul[@class="viewad-meta"]')78 if len(viewad_meta) == 2:79 viewad_meta_produce = viewad_meta[0]80 viewad_meta_contact = viewad_meta[1]81 viewad_meta_label_list = viewad_box.xpath('.//ul[@class="viewad-meta"]/li/label/text()')82 label_list = [label.replace('\r\n', '').replace(' ', '').replace('ï¼', '') for label in83 viewad_meta_label_list]84 self.check_keys_exists(label_list, go007_zf_map)85 viewad_meta_span_list = viewad_meta_produce.xpath('.//li/span/text()')86 address = None87 try:88 meta_suf = ''89 viewad_meta_contact = viewad_box.xpath('.//ul[@class="viewad-meta"]')[1]90 meta_list_pre = viewad_meta_contact.xpath('.//li[3]/span/a/text()')[0]91 meta_list_suf = viewad_meta_contact.xpath('.//li[3]/span/text()')92 for _meta_suf in meta_list_suf:93 meta_suf += _meta_suf.replace('\r\n', '').replace(' ', '')94 address = meta_list_pre + meta_suf95 except:96 pass97 viewad_meta_dic = {}98 try:99 name = re.search('unescape\("(.*?)\"\)', res).group(1).replace('%u', '\\u')100 name = urllib_parse.unquote(name.encode().decode('unicode-escape'))101 if not self.is_all_chinese(name):102 name = None103 except:104 name = None105 try:106 phone = re.search('unescape\("(\d+)', res).group(1)107 except:108 phone = e.xpath('.//input[@id="hfPhone"]/@value')[0]109 else:110 phone = None111 viewad_meta_span_list.extend([name, phone, address])112 # print(viewad_meta_span_list)113 for label in label_list:114 viewad_meta_dic.update({115 go007_zf_map.get(label): viewad_meta_span_list[label_list.index(label)]116 })117 parse_result_a.update({118 'viewad_box': viewad_meta_dic119 })120 if detail_view:121 detail_view = detail_view[0]122 dtlist = detail_view.xpath('./dd/div[@class="dllist"]/dl/dt/text()')123 dllist = detail_view.xpath('./dd/div[@class="dllist"]/dl/dd/text()')124 view_box = detail_view.xpath('./dd/div[@class="view_box"]/text()')125 container = detail_view.xpath('./dd/ul[@class="container"]/li/img/@src')126 # print(dtlist, '\n', dllist)127 d_dic = {}128 for dt in dtlist:129 if dt in go007_zf_map.keys():130 d_dic.update({go007_zf_map.get(dt): dllist[dtlist.index(dt)]})131 parse_result_a['detail_view']['detail_produce'] = d_dic132 if view_box:133 parse_result_a['detail_view']['view_box'] = view_box[0].replace('\r\n', '').replace(' ', '')134 parse_result_a['detail_view']['img_box'] = container135 # print('------\n', d_dic, '\n', view_box, '\n', container)136 # print(parse_result_a)137 parse_status = {'status': 1}138 except ParseFailureException as e:139 raise e('parse_html_a parse failure')140 parse_result_a.update(parse_status)141 return parse_result_a142 def parse_zf_detail_html_b(self, res):143 parse_result_b = {144 'parse_type': 'B',145 'viewad_header': {146 'view_title': None,147 'view_actions': None,148 },149 'nopic_box': {150 'msg_type': None,151 'source': None,152 'area': None,153 'price': None,154 'name': None,155 'phone': None,156 'address': None,157 'decoration_situation': None,158 'storey': None,159 'deposit': None,160 'layout': None,161 'orientation': None,162 'housing_estate': None,163 'view_box': None,164 },165 }166 e = etree.HTML(res)167 parse_status = {'status': -1}168 try:169 detail_top = e.xpath('.//div[@class="content"]/div[@class="left"]/div[@class="details_top"]')170 if detail_top:171 detail_top = detail_top[0]172 viewad_header = detail_top.xpath('./div[@class="viewad-header"]')173 no_pic = detail_top.xpath('./div[@class="no_pic"]')174 if viewad_header:175 viewad_header = viewad_header[0]176 view_title = viewad_header.xpath('./div[@class="viewad-title "]/h1/text()')177 view_actions = viewad_header.xpath('./div[@class="viewad-actions"]/span[1]/text()')178 if view_title:179 view_title = view_title[0].replace('\r\n', '').replace(' ', '')180 parse_result_b['viewad_header']['view_title'] = view_title181 if view_actions:182 parse_result_b['viewad_header']['view_actions'] = view_actions[0]183 # print(view_title, '\n', view_actions[0])184 if no_pic:185 no_pic = no_pic[0]186 nopic_dic = {}187 dtlist_box = no_pic.xpath('./div[@class="dllist"]/dl/dt/text()')188 dllist_box = no_pic.xpath('./div[@class="dllist"]/dl/dd/text()|\189 ./div[@class="dllist"]/dl/dd/em/text()')190 dllist = [dl.replace('\r\n', '').replace(' ', '') for dl in dllist_box if191 dl.replace('\r\n', '').replace(' ', '') != '']192 viewad = detail_top.xpath('./div[@class="no_pic"]/div[@class="viewad"]/ul[@class="viewad-meta"]')[0]193 # print(dtlist_box, '\n', dllist, '\n', viewad)194 address = ''195 viewad_label_list = viewad.xpath('./li/label/text()')196 label_list = [label.replace('\r\n', '').replace(' ', '') for label in viewad_label_list]197 viewad_span_pre = viewad.xpath('./li[3]/span/a/text()')198 for viewad_span in viewad_span_pre:199 viewad_span = viewad_span + '-'200 address += viewad_span201 span_list = viewad.xpath('./li[3]/span/text()')202 # print(span_list[-1])203 viewad_span_suf = span_list[-1].replace('\r\n', '').replace(' ', '')204 address += viewad_span_suf205 try:206 name = re.search('unescape\("(.*?)\"\)', res).group(1).replace('%u', '\\u')207 name = urllib_parse.unquote(name.encode().decode('unicode-escape'))208 if not self.is_all_chinese(name):209 name = None210 # print({'name': name})211 except:212 name = None213 try:214 phone = re.search('unescape\("(\d+)', res).group(1)215 except:216 phone = e.xpath('.//input[@id="hfPhone"]/@value')[0]217 else:218 phone = None219 nopic_dic.update({'name': name, 'phone': phone, 'address': address})220 if dtlist_box:221 self.check_keys_exists(dtlist_box, go007_zf_map)222 if dllist_box:223 for dt in dtlist_box:224 nopic_dic.update({go007_zf_map.get(dt): dllist[dtlist_box.index(dt)]})225 parse_result_b["nopic_box"] = nopic_dic226 parse_status = {'status': 1}227 except ParseFailureException as e:228 raise e('parse_html_a parse failure')229 parse_result_b.update(parse_status)230 return parse_result_b231 def parse_html(self, res):232 parse_result = {}233 result = {}234 if re.search('class="pic_box"', res):235 parse_result = self.parse_zf_detail_html_a(res)236 result.update({237 'header_link': None,238 'title': parse_result.get('viewad_header', {}).get('view_title'),239 'pubdate': parse_result.get('viewad_header', {}).get('view_actions', {}),240 'img_url': parse_result.get('img_url'),241 'msg_type': parse_result.get('viewad_box', {}).get('msg_type'),242 'source': parse_result.get('viewad_box', {}).get('source'),243 'area': parse_result.get('viewad_box', {}).get('area'),244 'price': parse_result.get('viewad_box', {}).get('price'),245 'name': parse_result.get('viewad_box', {}).get('name'),246 'phone': parse_result.get('viewad_box', {}).get('phone'),247 'address': parse_result.get('viewad_box', {}).get('address'),248 'decoration_situation': parse_result.get('detail_view', {}).get('detail_produce', {}).get(249 'decoration_situation'),250 'storey': parse_result.get('detail_view').get('detail_produce', {}).get('storey'),251 'deposit': parse_result.get('detail_view').get('detail_produce', {}).get('deposit'),252 'layout': parse_result.get('detail_view').get('detail_produce', {}).get('layout'),253 'orientation': parse_result.get('detail_view').get('detail_produce', {}).get('orientation'),254 'housing_estate': parse_result.get('detail_view').get('detail_produce', {}).get('housing_estate'),255 'view_box': parse_result.get('detail_view').get('view_box'),256 'img_box': parse_result.get('detail_view').get('img_box'),257 'status' : parse_result.get('status')258 })259 if re.search('class="no_pic"', res):260 parse_result = self.parse_zf_detail_html_b(res)261 result.update({262 'header_link': None,263 'title': parse_result.get('viewad_header',{}).get('view_title'),264 'pubdate': parse_result.get('viewad_header',{}).get('view_actions',{}),265 'img_url': parse_result.get('img_url'),266 'msg_type': parse_result.get('nopic_box',{}).get('msg_type'),267 'source': parse_result.get('nopic_box',{}).get('source'),268 'area': parse_result.get('nopic_box',{}).get('area'),269 'price': parse_result.get('nopic_box',{}).get('price'),270 'name': parse_result.get('nopic_box',{}).get('name'),271 'phone': parse_result.get('nopic_box',{}).get('phone'),272 'address': parse_result.get('nopic_box',{}).get('address'),273 'decoration_situation': parse_result.get('nopic_box',{}).get('decoration_situation'),274 'storey': parse_result.get('nopic_box').get('storey'),275 'deposit': parse_result.get('nopic_box').get('deposit'),276 'layout': parse_result.get('nopic_box').get('layout'),277 'orientation': parse_result.get('nopic_box').get('orientation'),278 'housing_estate': parse_result.get('nopic_box').get('housing_estate'),279 'view_box': parse_result.get('nopic_box').get('view_box'),280 'img_box': parse_result.get('nopic_box').get('img_box'),281 'status':parse_result.get('status')282 })283 return result284if __name__ == '__main__':285 import requests286 res_a = requests.get('http://www.go007.com/haikou/fangwuchuzu/714411d2b15c5ea1.htm').text287 res_b = requests.get('http://www.go007.com/beijing/fangwuchuzu/e11fd68dd51dea68.htm').text288 cjfc = ChengJiFangChan()289 # cjfc.parse_zf_detail_html_a(res_a)290 # cjfc.parse_zf_detail_html_b(res_b)291 print(cjfc.parse_html(res_b))292 {'parse_type': 'A', 'img_url': 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_0.jpg',293 'viewad_header': {'view_actions': '2020å¹´02æ11æ¥ 18:13', 'view_title': 'å¨ç¯å¢ä¼ç¾çæåºé'},294 'detail_view': {'view_box': '空æ°æ¸
æ°ï¼ç¯å¢ä¼ç¾ï¼å¨èææå室åé¢ï¼å¹³æ¿ï¼å¯å
»é¸¡?ç§èï¼ç¦»éä¸ä¸è¿ï¼é è¿ç»¿å满æ¥é£',295 'detail_produce': {'orientation': 'å', 'Rental_allocation': 'åº,çæ°´å¨,ç¬ç«å«çé´,é³å°,å¯åé¥',296 'decoration_situation': 'ç®åè£
ä¿®', 'housing_estate': 'äºé¾éèæ'},297 'img_box': ['http://img.go007.com/2020/02/11/714411d2b15c5ea1_1.jpg',298 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_2.jpg',299 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_3.jpg',300 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_4.jpg']},301 'viewad_box': {'msg_type': 'æ¿äº§ æ¿å±åºç§', 'source': '个人', 'phone': '13518066257', 'address': 'æµ·å£-ç¼å±±_èææå室',302 'name': 'å¿å', 'price': ' 500å
', 'area': '120ã¡'}}303 {'viewad_header': {'view_actions': '2019å¹´09æ29æ¥ 16:40', 'view_title': 'æ¿ä¸ç´ç§åä¸ç¯å½å±å·¦å®¶åºä¸å±
室'}, 'parse_type': 'B',304 'nopic_box': {'price': '5700å
', 'storey': '8å±', 'Rental_allocation': 'åº,æ²å,çæ°´å¨,çµèæ¡æ¤
,è¡£æ,çµè§,空è°,å°ç®±,æ´è¡£æº,ææ°',305 'housing_estate': '左家åºåé', 'layout': '1室1å
1å«', 'phone': None, 'msg_type': 'æ¿äº§æ¿å±åºç§', 'source': '个人',306 'decoration_situation': 'ç®åè£
ä¿®', 'deposit': 'æ¼ä¸ä»ä¸', 'address': 'å京-æé³åº-ä¸å
æ¡¥-', 'name': None,307 'area': '46ã¡'}}308{'storey': None, 'view_title': 'å¨ç¯å¢ä¼ç¾çæåºé', 'Rental_allocation': 'åº,çæ°´å¨,ç¬ç«å«çé´,é³å°,å¯åé¥', 'phone': '13518066257',309 'source': '个人', 'orientation': 'å', 'title': None, 'deposit': None, 'view_actions': '2020å¹´02æ11æ¥ 18:13',310 'header_link': None, 'address': 'æµ·å£-ç¼å±±_èææå室', 'name': 'å¿å', 'price': ' 500å
', 'decoration_situation': 'ç®åè£
ä¿®',311 'housing_estate': 'äºé¾éèæ', 'pubdate': None, 'area': '120ã¡',312 'img_url': 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_0.jpg', 'msg_type': 'æ¿äº§ æ¿å±åºç§',313 'img_box': ['http://img.go007.com/2020/02/11/714411d2b15c5ea1_1.jpg',314 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_2.jpg',315 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_3.jpg',316 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_4.jpg'],317 'view_box': '空æ°æ¸
æ°ï¼ç¯å¢ä¼ç¾ï¼å¨èææå室åé¢ï¼å¹³æ¿ï¼å¯å
»é¸¡?ç§èï¼ç¦»éä¸ä¸è¿ï¼é è¿ç»¿å满æ¥é£', 'layout': None}318{'storey': None, 'pubdate': '2020å¹´02æ11æ¥ 18:13', 'phone': '13518066257', 'name': 'å¿å', 'address': 'æµ·å£-ç¼å±±_èææå室',319 'orientation': 'å', 'msg_type': 'æ¿äº§ æ¿å±åºç§', 'img_url': 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_0.jpg',320 'header_link': None, 'decoration_situation': 'ç®åè£
ä¿®', 'housing_estate': 'äºé¾éèæ', 'deposit': None, 'layout': None,321 'area': '120ã¡', 'price': ' 500å
', 'view_box': '空æ°æ¸
æ°ï¼ç¯å¢ä¼ç¾ï¼å¨èææå室åé¢ï¼å¹³æ¿ï¼å¯å
»é¸¡?ç§èï¼ç¦»éä¸ä¸è¿ï¼é è¿ç»¿å满æ¥é£',322 'img_box': ['http://img.go007.com/2020/02/11/714411d2b15c5ea1_1.jpg',323 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_2.jpg',324 'http://img.go007.com/2020/02/11/714411d2b15c5ea1_3.jpg',...
run_start.py
Source:run_start.py
1import logging2import os3import sys4import argparse5from datetime import datetime6import time7import traceback8# rcdb stuff9import rcdb10from rcdb.log_format import BraceMessage as Lf11from rcdb import ConfigurationProvider, UpdateReasons12from rcdb import UpdateContext, UpdateReasons, DefaultConditions13from rcdb.provider import RCDBProvider14from rcdb.model import ConditionType, Condition15# parity stuff16from parity_rcdb import parity_coda_parser17from parity_rcdb import ParityConditions18from parity_rcdb.parity_coda_parser import ParityCodaRunLogParseResult19# helper script20import epics_helper21"""22Update script at run start23controlSessions.xml24Things to consider for the future:251) adding interprocess_lock: to avoid duplicated records and ensure only one such process is running26"""27# Test flag, it will print out parse result but no udpate to DB28test_mode = False29def get_usage():30 return """31 Usage:32 minimal:33 run_start.py <session_xml_file>34 35 run_start.py <session_xml_file> -c <db_connection_string> --update=coda,epics --reason=[start, update, end] --exp=PREX236 37 example:38 run_start.py controlSessions.xml --exp=PREX239 40 <db_connection_string> - is optional. But if it is not set, RCDB_CONNECTION environment variable should be set41 """42 43def parse_start_run_info():44 script_start_clock = time.clock()45 script_start_time = time.time()46 log = logging.getLogger('pvdb') # create run configuration standard logger47 log.addHandler(logging.StreamHandler(sys.stdout)) # add console output for logger48 log.setLevel(logging.INFO) # DEBUG: print everything. Changed to logging. INFO for less output49 parser = argparse.ArgumentParser(description= "Update PVDB")50# parser.add_argument("config_xml_file", type=str, help="full path to configID.xml file")51 parser.add_argument("session_xml_file", type=str, help="full path to controlSessions.xml file")52 parser.add_argument("-v", "--verbose", help="increase output verbosity", action="store_true")53 parser.add_argument("--update", help="Comma separated, modules to update such as coda,epics", default="coda,epics")54 parser.add_argument("--reason", help="Reason of the udpate: 'start', 'udpate', 'end' or ''", default="start")55 parser.add_argument("--exp", help="Experiment name", default="CREX")56 parser.add_argument("-c", "--connection", help="connection string (eg, mysql://pvdb@localhost/pvdb)")57 args = parser.parse_args()58 # Set log level59 log.setLevel(logging.DEBUG if args.verbose else logging.INFO)60 # coda xml files61# config_xml_file = args.config_xml_file62# log.debug(Lf("config_xml_file = '{}'", config_xml_file))63 session_xml_file = args.session_xml_file64 log.debug(Lf("session_xml_file = '{}' ", session_xml_file))65 66 # Connection67 if args.connection:68 con_string = args.connection69 elif "RCDB_CONNECTION" in os.environ:70 con_string = os.environ["RCDB_CONNECTION"]71 else:72 print ("ERROR! RCDB_CONNECTION is not set and is not given as a script parameter (-c)")73 parser.print_help()74 sys.exit(2)75 log.debug(Lf("con_string = '{}'", con_string))76 # What to update 77 update_parts = [] 78 if args.update: 79 update_parts = args.update.split(',') 80 log.debug(Lf("update_parts = {}", update_parts))81 # Update reason82 update_reason = args.reason83 log.debug(Lf("update_reason = '{}'", update_reason))84 # Open DB connection85 db = ConfigurationProvider(con_string)86 87 # Create update context88 update_context = rcdb.UpdateContext(db, update_reason)89 # Parse coda files and save to DB90 log.debug(Lf("Parsing coda__xml_files='{}'", session_xml_file))91 coda_parse_result = parity_coda_parser.parse_start_run_data(session_xml_file)92 run_number = coda_parse_result.run_number93 # >oO DEBUG log message94 now_clock = time.clock()95 db.add_log_record("", "Start Run Script: Parsed xml_files='{}'. run='{}'"96 .format(session_xml_file, run_number), run_number)97 # Parse runstart.info (Run type, comment)98 dict_info = parity_coda_parser.runinfo_parser()99 if bool(dict_info["Run"]["type"]):100 coda_parse_result.run_type = dict_info["Run"]["type"]101 if bool(dict_info["comment"]["text"]):102 coda_parse_result.user_comment = dict_info["comment"]["text"]103 if bool(dict_info["parity"]["bmw"]):104 coda_parse_result.bmw = dict_info["parity"]["bmw"]105 if bool(dict_info["parity"]["feedback"]):106 coda_parse_result.feedback = dict_info["parity"]["feedback"]107 # Set experiment name108 coda_parse_result.experiment_name = args.exp109 # Coda conditions to DB110 if "coda" in update_parts:111 log.debug(Lf("Adding coda conditions to DB", ))112 if test_mode:113 print "Experiment name:\t", coda_parse_result.experiment_name114 print "Run number:\t", coda_parse_result.run_number115 print "Start time:\t", coda_parse_result.start_time116 print "Run config:\t", coda_parse_result.run_config117 print "Run type:\t", coda_parse_result.run_type118 print "Comment:\t", coda_parse_result.user_comment119 print "BMW:\t", coda_parse_result.bmw120 print "Feedback:\t", coda_parse_result.feedback121 else:122 update_parity_coda_conditions(update_context, coda_parse_result)123 else:124 log.debug(Lf("Skipping to add coda conditions to DB. Use --update=...,coda to update it", ))125 """126 # Do we want to save files to DB?127 log.debug(Lf("Adding coda_xml_log_file to DB", ))128 db.add_configuration_file(run_number, coda_xml_log_file, overwrite=True, importance=ConfigurationFile.IMPORTANCE_HIGH)129 """130 131 # EPICS Update132 # Get EPICS variables133 epics_start_clock = time.clock()134 if test_mode:135 conditions = epics_helper.get_run_conds()136 print conditions137 else:138 if 'epics' in update_parts and run_number:139 # noinspection PyBroadException140 try:141 conditions = epics_helper.update_db_conds(db, run_number, update_reason)142 epics_end_clock = time.clock()143 # >oO DEBUG log message144 if "beam_current" in conditions:145 db.add_log_record("",146 "Update epics. beam_current:'{}', time: '{}'"147 .format(conditions["beam_current"], datetime.now()), run_number)148 149 except Exception as ex:150 log.warn("update_epics.py failure. Impossible to run the script. Internal exception is:\n" + str(ex))151 epics_end_clock = time.clock()152 # >oO DEBUG log message153 db.add_log_record("",154 "ERROR update epics. Error type: '{}' message: '{}' trace: '{}' "155 "||epics_clocks:'{}' clocks:'{}' time: '{}'"156 .format(type(ex), ex.message, traceback.format_exc(),157 epics_end_clock - epics_start_clock, epics_end_clock - script_start_clock,158 datetime.now()), run_number)159 160 log.debug("End of update")161 # >oO DEBUG log message162 now_clock = time.clock()163 if not test_mode:164 db.add_log_record("",165 "End of update. Script proc clocks='{}', wall time: '{}', datetime: '{}'"166 .format(now_clock - script_start_clock,167 time.time() - script_start_time,168 datetime.now()), run_number)169def update_parity_coda_conditions(context, parse_result):170 # create run configuration standard logger171 log = logging.getLogger('pvdb.update_coda') 172 # Some assertions in the beginning173 assert isinstance(parse_result, ParityCodaRunLogParseResult)174 assert isinstance(context, UpdateContext)175 assert isinstance(context.db, RCDBProvider)176 db = context.db177 178 if parse_result.run_number is None:179 log.warn("parse_result.run_number is None. (!) Run. Number. Is. None!!!")180 return181 182 """183 # enable later184 if context.reason == UpdateReasons.END and not db.get_run(parse_result.run_number):185 log.info(Lf("Run '{}' is not found in DB. But the update reason is end of run. "186 "Considering there where no GO. Only prestart and then Stop ", parse_result.run_number))187 return188 """189 run = db.create_run(parse_result.run_number)190 conditions = []191 # Experiment name192 if parse_result.experiment_name is not None:193 conditions.append((ParityConditions.EXPERIMENT, parse_result.experiment_name))194 # Arm flag (default set to 0)195 conditions.append((ParityConditions.ARM_FLAG, 0))196 # Run type condition197 if parse_result.run_type is not None:198 conditions.append((DefaultConditions.RUN_TYPE, parse_result.run_type))199 # Session 200 if parse_result.coda_session is not None:201 conditions.append((DefaultConditions.SESSION, parse_result.coda_session))202 # config name203 if parse_result.run_config is not None:204 conditions.append((DefaultConditions.RUN_CONFIG, parse_result.run_config))205 # slug number206 if parse_result.slug is not None:207 conditions.append((ParityConditions.SLUG, parse_result.slug))208 #These need to be udpated by Run END209 # Set the run as not properly finished (We hope that the next section will210 if parse_result.has_run_end is not None:211 conditions.append((DefaultConditions.IS_VALID_RUN_END, parse_result.has_run_end))212 # The number of events in the run213 if parse_result.event_count is not None:214 conditions.append((DefaultConditions.EVENT_COUNT, parse_result.event_count))215 216 # Daq comment by user217 if parse_result.user_comment is not None:218 conditions.append((DefaultConditions.USER_COMMENT, parse_result.user_comment))219 if parse_result.bmw is not None:220 conditions.append((ParityConditions.BMW, parse_result.bmw))221 if parse_result.feedback is not None:222 conditions.append((ParityConditions.FEEDBACK, parse_result.feedback))223 # Run prestart time224 if parse_result.prestart_time is not None:225 conditions.append((ParityConditions.RUN_PRESTART_TIME, parse_result.prestart_time))226 # Run length227 if parse_result.has_run_end == True and parse_result.has_run_start == True:228 total_run_time = datetime.strptime(parse_result.end_time, "%Y-%m-%d %H:%M:%S") - datetime.strptime(parse_result.start_time, "%Y-%m-%d %H:%M:%S")229 conditions.append((DefaultConditions.RUN_LENGTH, total_run_time.seconds))230 #start and end time is read from run table, so no need to add these conditions here231# conditions.append((DefaultConditions.RUN_START_TIME, datetime.strptime(parse_result.start_time,"%m/%d/%y %H:%M:%S")))232# conditions.append((DefaultConditions.RUN_END_TIME, datetime.strptime(parse_result.end_time, "%m/%d/%y %H:%M:%S")))233 """234 # Conditions not added currently235 # Filename of the last evio file written by CODA ER236 if parse_result.coda_last_file is not None:237 conditions.append(('evio_last_file', parse_result.evio_last_file))238 # The number of evio files written by CODA Event Recorder239 if parse_result.coda_files_count is not None:240 conditions.append(('evio_files_count', parse_result.evio_files_count))241 """242 # SAVE CONDITIONS243 db.add_conditions(run, conditions, replace=True)244 log.info(Lf("update_coda: Saved {} conditions to DB", len(conditions)))245 # Start and end times246 if parse_result.start_time is not None:247 run.start_time = parse_result.start_time # Time of the run start248 log.info(Lf("Run start time is {}", parse_result.start_time))249 if parse_result.end_time is not None:250 run.end_time = parse_result.end_time # Time of the run end251 log.info(Lf("Run end time is {}. Set from end_time record", parse_result.end_time))252 """253 # update_time is not currently included in ParityCodaRunLogParseResult254 else:255 if parse_result.update_time is not None:256 run.end_time = parse_result.update_time # Fallback, set time when the coda log file is written as end time257 log.info(Lf("Run end time is {}. Set from update_time record", parse_result.update_time))258 """259 db.session.commit() # Save run times260if __name__== '__main__':...
card2vec.py
Source:card2vec.py
1# -*- coding: utf-8 -*-2import json3#import MeCab4from gensim.models import doc2vec5import os6TARGET_GAME_NAME = "all"7def load_json(target_game_name,file_name):8 # ã«ã¼ãåã¨ã«ã¼ãããã¹ãã®å
¥åãã¼ã¿ä½æ9 names = []10 text = ""11 texts = []12 # Mecabã®åºåãåãã¡æ¸ãã«æå®13 #mecab = MeCab.Tagger("-Owakati")14 json_path = target_game_name + ".json"15 #import stanza16 #nlp = stanza.Pipeline(lang='en', processors='tokenize,lemma')17 # ã«ã¼ãã®ããã¹ããå½¢æ
ç´ è§£æããåãã¡æ¸ããããã®ãæ¹è¡åºåãã§ä¸ã¤ã®stringã«ãã18 with open(json_path, "r") as file:19 card_dict = json.load(file)20 card_name_list = []21 for card_id in card_dict:22 card_name_list.append(card_dict[card_id]["name_"])23 for card_id in card_dict:24 card = card_dict[card_id]25 #print(card.keys())26 if card["name_"] not in names:27 names.append(card["name_"])28 pp = str(card["pp_"]) #"pp: " +29 card_type = card["type_"] #"type: " +30 # pp = "pp: " + str(card["pp_"])31 # craft = "craft: " + card["craft_"].replace("craft","")32 # card_type = "type: " + card["type_"]33 # base_atk = "baseAtk: -"34 # base_def = "baseDef: -"35 # evo_atk = "evoAtk : -"36 # evo_def = "evoDef: -"37 # if "Follower" in card_type:38 # base_atk = "baseAtk: " + str(card["baseAtk_"])39 # base_def = "baseDef: " + str(card["baseDef_"])40 # evo_atk = "evoAtk: " + str(card["evoAtk_"])41 # evo_def = "evoDef: " + str(card["evoDef_"])42 base_effect = card["baseEffect_"] #"baseEffect: " +43 base_effect = base_effect.replace("\n","$")44 evo_effect = card["evoEffect_"] # "evoEffect: " +45 evo_effect = evo_effect.replace("\n","$")46 if "Same as the unevolved form" in evo_effect:47 evo_effect = card["baseEffect_"]48 if "----------" in evo_effect:49 evo_effect = evo_effect.split("----------")[1]50 except_words = ["Enhance","Accelerate","Crystallize","Fanfare","Invocation",\51 "When this follower comes into play"]52 for except_word in except_words:53 if except_word in evo_effect:54 evo_effect = evo_effect.split("$")55 evo_effect = [sentence for sentence in evo_effect if except_word not in sentence]56 if len(evo_effect) > 0:57 evo_effect = "$".join(evo_effect)58 else:59 evo_effect = "-"60 default_text = card["baseEffect_"]61 if "Fanfare" in default_text and "Last Words" in default_text:62 evo_effect += "Last Words:" + default_text.split("Last Words:")[-1]63 evo_effect = "evoEffect: " + evo_effect64 base_effect = base_effect.replace("$", " ")65 evo_effect = evo_effect.replace("$"," ")66 parse_result = base_effect + " " + evo_effect67 parse_result = parse_result.replace(".", " . ")68 parse_result = parse_result.replace("/", " / ")69 parse_result = parse_result.replace(":", " : ")70 parse_result = parse_result.lower()71 for name in card_name_list:72 if name.lower() in parse_result:73 new_name = name.replace(" ","")74 new_name = new_name.replace(", ",",")75 parse_result = parse_result.replace(name.lower(),new_name)76 parse_result = parse_result.replace(", ", " , ")77 parse_result = pp + "\n" + card_type + "\n" + parse_result78 #doc = nlp(parse_result)79 #parse_result = " ".join([word.lemma for sent in doc.sentences for word in sent.words])80 # parse_result = craft + " " + card_type + "\n"81 # parse_result += pp + "\n"82 # parse_result += base_atk + " " + base_def + "\n"83 # parse_result += base_effect + "\n"84 # parse_result += evo_atk + " " + evo_def + "\n"85 # parse_result += evo_effect86 #mecab_result = mecab.parse(card["text"])87 if parse_result is False:88 text += "\n"89 texts.append("")90 else:91 #text += parse_result + "\n"92 texts.append(parse_result)93 text += parse_result.replace("\n"," ") + "\n"94 #names.append(card["name_"]+"(EVO)")95 #parse_result=card["evoEffect_"]96 #parse_result = parse_result.replace("\n"," ")97 #if parse_result is False:98 # text += "\n"99 # texts.append("")100 #else:101 # text += parse_result + "\n"102 # texts.append(card["evoEffect_"])103 TARGET = target_game_name +"_"+file_name+ ".txt"104 if os.path.isfile(TARGET) is False:105 with open(TARGET, "w") as file:106 file.write(text)107 return names, texts108def generate_doc2vec_model(target_game_name,size=300,window=8):109 print("Training Start")110 # ã«ã¼ãããã¹ãèªã¿è¾¼ã¿111 card_text = doc2vec.TaggedLineDocument(target_game_name + ".txt")112 # å¦ç¿113 model = doc2vec.Doc2Vec(card_text, size=size, window=window, min_count=1,114 workers=4, iter=400, dbow_words=1, negative=10,dm=0)115 # ã¢ãã«ã®ä¿å116 model.save(target_game_name + ".model")117 print("Training Finish")118 return model119if __name__ == '__main__':120 import argparse121 parser = argparse.ArgumentParser(description='Card2Vecå¦ç¿ã³ã¼ã')122 parser.add_argument('--vector_size', help='ã«ã¼ããã¯ãã«ã®ãµã¤ãº', type=int, default=300)123 parser.add_argument('--window_size', help='çªãµã¤ãº', type=int, default=8)124 parser.add_argument('-visualize',help='ãã¯ãã«ã®å¯è¦å')125 args = parser.parse_args()126 names, texts = load_json(TARGET_GAME_NAME,"{}_{}".format(args.vector_size,args.window_size))127 128 target_file_name = "{}_{}_{}".format(TARGET_GAME_NAME,129 args.vector_size,args.window_size)130 if os.path.isfile(target_file_name + ".model") is True:131 model = doc2vec.Doc2Vec.load(TARGET_GAME_NAME + ".model")132 else:133 model = generate_doc2vec_model(target_file_name,size=args.vector_size,window=args.window_size)134 if args.visualize is not None:135 init_word = input("input initial word:")136 positive_word = input("input positive word:")137 negatice_word = ""#("input negative word:")138 query = model.most_similar(positive=[init_word,positive_word])#,negative=[negatice_word])139 print(query)140 #print(names[card_index])141 #print(texts[card_index])142 #print(model.docvecs[card_index])143 TARGET_CARD_NAME = input("input card_name:")144 card_index = names.index(TARGET_CARD_NAME)145 print(model.docvecs[card_index])146 print(model.docvecs["Goblin"])147 reverse_flg = input("reverse?(y/n):") == "y" 148 # é¡ä¼¼ã«ã¼ãã¨é¡ä¼¼åº¦ã®ã¿ãã«ï¼é¡ä¼¼åº¦ä¸ä½10件ï¼ã®ãªã¹ããåãåã149 similar_docs = model.docvecs.most_similar(card_index,topn=len(names))150 print("name_len:",len(names),"txt_len:",len(texts))151 print(names[card_index])152 print(texts[card_index])153 print("--------------------is similar to--------------------")154 if reverse_flg:155 similar_docs = list(reversed(similar_docs))156 for similar_doc in similar_docs[0:10]:157 #print("similar_doc:",similar_doc)158 print(names[similar_doc[0]] + " " + str(similar_doc[1]))...
test_connection_string_parser.py
Source:test_connection_string_parser.py
1#-------------------------------------------------------------------------2# Copyright (c) Microsoft Corporation. All rights reserved.3# Licensed under the MIT License. See License.txt in the project root for4# license information.5#--------------------------------------------------------------------------6import os7import pytest8from azure.servicebus import (9 ServiceBusConnectionStringProperties,10 parse_connection_string,11)12from devtools_testutils import AzureMgmtTestCase13class ServiceBusConnectionStringParserTests(AzureMgmtTestCase):14 def test_sb_conn_str_parse_cs(self, **kwargs):15 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='16 parse_result = parse_connection_string(conn_str)17 assert parse_result.endpoint == 'sb://resourcename.servicebus.windows.net/'18 assert parse_result.fully_qualified_namespace == 'resourcename.servicebus.windows.net'19 assert parse_result.shared_access_key_name == 'test'20 assert parse_result.shared_access_key == 'THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='21 def test_sb_conn_str_parse_sas_and_shared_key(self, **kwargs):22 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX=;SharedAccessSignature=THISISASASXXXXXXX='23 with pytest.raises(ValueError) as e:24 parse_result = parse_connection_string(conn_str)25 assert str(e.value) == 'Only one of the SharedAccessKey or SharedAccessSignature must be present.'26 27 def test_sb_parse_malformed_conn_str_no_endpoint(self, **kwargs):28 conn_str = 'SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='29 with pytest.raises(ValueError) as e:30 parse_result = parse_connection_string(conn_str)31 assert str(e.value) == 'Connection string is either blank or malformed.'32 def test_sb_parse_malformed_conn_str_no_endpoint_value(self, **kwargs):33 conn_str = 'Endpoint=;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='34 with pytest.raises(ValueError) as e:35 parse_result = parse_connection_string(conn_str)36 assert str(e.value) == 'Connection string is either blank or malformed.'37 def test_sb_parse_malformed_conn_str_no_netloc(self, **kwargs):38 conn_str = 'Endpoint=MALFORMED;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='39 with pytest.raises(ValueError) as e:40 parse_result = parse_connection_string(conn_str)41 assert str(e.value) == 'Invalid Endpoint on the Connection String.'42 def test_sb_parse_conn_str_sas(self, **kwargs):43 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessSignature=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='44 parse_result = parse_connection_string(conn_str)45 assert parse_result.endpoint == 'sb://resourcename.servicebus.windows.net/'46 assert parse_result.fully_qualified_namespace == 'resourcename.servicebus.windows.net'47 assert parse_result.shared_access_signature == 'THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='48 assert parse_result.shared_access_key_name == None49 def test_sb_parse_conn_str_whitespace_trailing_semicolon(self, **kwargs):50 conn_str = ' Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessSignature=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX=; '51 parse_result = parse_connection_string(conn_str)52 assert parse_result.endpoint == 'sb://resourcename.servicebus.windows.net/'53 assert parse_result.fully_qualified_namespace == 'resourcename.servicebus.windows.net'54 assert parse_result.shared_access_signature == 'THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='55 assert parse_result.shared_access_key_name == None56 def test_sb_parse_conn_str_sas_trailing_semicolon(self, **kwargs):57 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessSignature=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX=;'58 parse_result = parse_connection_string(conn_str)59 assert parse_result.endpoint == 'sb://resourcename.servicebus.windows.net/'60 assert parse_result.fully_qualified_namespace == 'resourcename.servicebus.windows.net'61 assert parse_result.shared_access_signature == 'THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='62 assert parse_result.shared_access_key_name == None63 def test_sb_parse_conn_str_no_keyname(self, **kwargs):64 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='65 with pytest.raises(ValueError) as e:66 parse_result = parse_connection_string(conn_str)67 assert str(e.value) == 'Connection string must have both SharedAccessKeyName and SharedAccessKey.'68 def test_sb_parse_conn_str_no_key(self, **kwargs):69 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKeyName=Test'70 with pytest.raises(ValueError) as e:71 parse_result = parse_connection_string(conn_str)72 assert str(e.value) == 'Connection string must have both SharedAccessKeyName and SharedAccessKey.'73 def test_sb_parse_conn_str_no_key_or_sas(self, **kwargs):74 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/'75 with pytest.raises(ValueError) as e:76 parse_result = parse_connection_string(conn_str)77 assert str(e.value) == 'At least one of the SharedAccessKey or SharedAccessSignature must be present.'78 def test_sb_parse_malformed_conn_str_lowercase_endpoint(self, **kwargs):79 conn_str = 'endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKeyName=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='80 with pytest.raises(ValueError) as e:81 parse_result = parse_connection_string(conn_str)82 assert str(e.value) == 'Connection string is either blank or malformed.'83 def test_sb_parse_malformed_conn_str_lowercase_sa_key_name(self, **kwargs):84 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;sharedaccesskeyname=test;SharedAccessKey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='85 with pytest.raises(ValueError) as e:86 parse_result = parse_connection_string(conn_str)87 assert str(e.value) == 'Connection string must have both SharedAccessKeyName and SharedAccessKey.'88 def test_sb_parse_malformed_conn_str_lowercase_sa_key_name(self, **kwargs):89 conn_str = 'Endpoint=sb://resourcename.servicebus.windows.net/;SharedAccessKeyName=test;sharedaccesskey=THISISATESTKEYXXXXXXXXXXXXXXXXXXXXXXXXXXXX='90 with pytest.raises(ValueError) as e:91 parse_result = parse_connection_string(conn_str)...
Playwright tutorial
LambdaTest’s Playwright tutorial will give you a broader idea about the Playwright automation framework, its unique features, and use cases with examples to exceed your understanding of Playwright testing. This tutorial will give A to Z guidance, from installing the Playwright framework to some best practices and advanced concepts.
Chapters:
- What is Playwright : Playwright is comparatively new but has gained good popularity. Get to know some history of the Playwright with some interesting facts connected with it.
- How To Install Playwright : Learn in detail about what basic configuration and dependencies are required for installing Playwright and run a test. Get a step-by-step direction for installing the Playwright automation framework.
- Playwright Futuristic Features: Launched in 2020, Playwright gained huge popularity quickly because of some obliging features such as Playwright Test Generator and Inspector, Playwright Reporter, Playwright auto-waiting mechanism and etc. Read up on those features to master Playwright testing.
- What is Component Testing: Component testing in Playwright is a unique feature that allows a tester to test a single component of a web application without integrating them with other elements. Learn how to perform Component testing on the Playwright automation framework.
- Inputs And Buttons In Playwright: Every website has Input boxes and buttons; learn about testing inputs and buttons with different scenarios and examples.
- Functions and Selectors in Playwright: Learn how to launch the Chromium browser with Playwright. Also, gain a better understanding of some important functions like “BrowserContext,” which allows you to run multiple browser sessions, and “newPage” which interacts with a page.
- Handling Alerts and Dropdowns in Playwright : Playwright interact with different types of alerts and pop-ups, such as simple, confirmation, and prompt, and different types of dropdowns, such as single selector and multi-selector get your hands-on with handling alerts and dropdown in Playright testing.
- Playwright vs Puppeteer: Get to know about the difference between two testing frameworks and how they are different than one another, which browsers they support, and what features they provide.
- Run Playwright Tests on LambdaTest: Playwright testing with LambdaTest leverages test performance to the utmost. You can run multiple Playwright tests in Parallel with the LammbdaTest test cloud. Get a step-by-step guide to run your Playwright test on the LambdaTest platform.
- Playwright Python Tutorial: Playwright automation framework support all major languages such as Python, JavaScript, TypeScript, .NET and etc. However, there are various advantages to Python end-to-end testing with Playwright because of its versatile utility. Get the hang of Playwright python testing with this chapter.
- Playwright End To End Testing Tutorial: Get your hands on with Playwright end-to-end testing and learn to use some exciting features such as TraceViewer, Debugging, Networking, Component testing, Visual testing, and many more.
- Playwright Video Tutorial: Watch the video tutorials on Playwright testing from experts and get a consecutive in-depth explanation of Playwright automation testing.
Run Playwright Python automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
