How to use xpath_class method in SeleniumBase

Best Python code snippet using SeleniumBase

mongo_query.py

Source:mongo_query.py

1from src.axes.both_child_descendant import both_child_descendant2from src.axes.only_child_axes_with_condition import only_child_axes_with_condition3from src.axes.only_child_axes import only_child_axes4from src.axes.only_descendant_axes import only_descendant_axes5from src.axes.only_child_axes_with_star import only_child_axes_with_star6from src.axes.only_descendant_axes_with_star import only_descendant_axes_with_star7from src.tokenizer import tokenize 8def xPath_classifier(xpath_tokenizer_lst):9    '''10    Symbols Identifier11    symbol_1: /12    symbol_2: [13    symbol_3: ]14    symbol_4: | 15    symbol_5: >16    symbol_6: <17    symbol_7: =18    symbol_8: *19    20    Words Identifier21    word_1: child::22    word_2: descendant::23    word_3: attribute::24    word_4: and25    word_5: or26    word_6: count27    word_7 = 'sum'28    others: anything that i haven thought of29    '''30    symbol_1 = '/'31    symbol_2 = '['32    symbol_3 = ']'33    symbol_4 = '|' 34    symbol_5 = '>'35    symbol_6 = '<'36    symbol_7 = '='37    symbol_8 = '*'38    symbol_9 = '//'39    40    symbol_1_count = 0 41    symbol_2_count = 0 42    symbol_3_count = 0 43    symbol_4_count = 0 44    symbol_5_count = 0 45    symbol_6_count = 0 46    symbol_7_count = 0 47    symbol_8_count = 0 48    symbol_9_count = 049    for x in xpath_tokenizer_lst:50        if x == symbol_1:51            symbol_1_count += 152        if x == symbol_2:53            symbol_2_count += 154        if x == symbol_3:55            symbol_3_count += 156        if x == symbol_4:57            symbol_4_count += 158        if x == symbol_5:59            symbol_5_count += 160        if x == symbol_6:61            symbol_6_count += 162        if x == symbol_7:63            symbol_7_count += 164        if x == symbol_8:65            symbol_8_count += 166        if x == symbol_9:67            symbol_9_count += 1  68    if (symbol_1_count > 0) and (symbol_2_count == 0) and (symbol_4_count == 0) and (symbol_5_count == 0) and (symbol_6_count == 0) and (symbol_7_count == 0) and (symbol_8_count == 0) and (symbol_9_count == 0):69        return "only_child_axes"70    if (symbol_1_count == 0) and (symbol_9_count > 0) and  (symbol_2_count == 0) and (symbol_4_count == 0) and (symbol_5_count == 0) and (symbol_6_count == 0) and (symbol_7_count == 0) and (symbol_8_count == 0):71        return "only_descendant_axes"72    if (symbol_1_count > 0) and (symbol_9_count > 0) and (symbol_2_count == 0) and (symbol_4_count == 0) and (symbol_5_count == 0) and (symbol_6_count == 0) and (symbol_7_count == 0) and (symbol_8_count == 0):73        return "both_child_descendant"74    if (symbol_1_count > 0) and (symbol_9_count == 0) and (symbol_2_count > 0) and (symbol_8_count == 0):75        return "only_child_axes_with_condition"76    if (symbol_1_count > 0) and (symbol_2_count == 0) and (symbol_8_count > 0) and (symbol_9_count == 0):77        return "only_child_axes_with_star"78    if (symbol_1_count == 0) and (symbol_2_count == 0) and (symbol_8_count > 0) and (symbol_9_count > 0):79        return "only_descendant_axes_with_star"80    81    return "Invalid Query"82VALID_SPECIAL_CHARACTER = ['/', '//', '[', ']', '*', '@', '<', '>', '>=', '<=', '=']83def checkValidXPath(xPath, schema):84    tokens = tokenize(xPath)85    if '/' not in xPath:86        return False87    tokens_strip_empty = list(filter(lambda a: a != '', tokens))88    89    for token in tokens_strip_empty:90        isValid = False91        for s in schema:92            if token in s.split('.') or token in VALID_SPECIAL_CHARACTER or token[0] == '\'' :93                isValid = True94        95    if not isValid:96        return False97    return True98    99def mongo_query(xPath, schema):100    isValidXPath = checkValidXPath(xPath, schema)101    102    xPath_class = xPath_classifier(tokenize(xPath))103    if isValidXPath:104        if xPath_class == "only_child_axes":105            return only_child_axes(xPath, schema)106        if xPath_class == "only_descendant_axes":107            return only_descendant_axes(xPath, schema)108        if xPath_class == "both_child_descendant":109            return both_child_descendant(xPath, schema)110        if xPath_class == "only_child_axes_with_condition":111            return only_child_axes_with_condition(xPath, schema)112        if xPath_class == "only_child_axes_with_star":113            return only_child_axes_with_star(xPath, schema)114        if xPath_class == "only_descendant_axes_with_star":115            return only_descendant_axes_with_star(xPath, schema)116        if xPath_class == 'Invalid Query':117            return "Invalid Syntax"118    else:119        return 'Invalid Syntax', 'Invalid Syntax'...

CapesQualisScraper.py

Source:CapesQualisScraper.py

1#!/usr/bin/python32from selenium import webdriver3from time import sleep4from time import time5import pandas as pd6from math import ceil7# Capes qualis data url address 8url = 'https://sucupira.capes.gov.br/sucupira/public/consultas/coleta/veiculoPublicacaoQualis/listaConsultaGeralPeriodicos.xhtml'9# try:	10# 	res = requests.get(url) # get data from address11# 	res.raise_for_status() # test if connection is successful12# except (requests.exceptions.HTTPError, MissingSchema):13# 	print('\nYou have entered an incorrect address.')14# 	sys.exit()15# launch browser16browser = webdriver.Chrome()17# open the url on the browser18browser.get(url)19# select option: CLASSIFICAÃÃES DE PERIÃDICOS QUADRIÃNIO 2013-201620browser.find_element_by_xpath('//*[@id="form:evento"]/option[2]').click()21# select option: MATEMÃTICA PROBABILIDADE E ESTATÃSTICA22browser.find_element_by_xpath('//*[@id="form:area"]/option[36]').click()23# include all previous area options24browser.find_element_by_xpath('//*[@id="form:adicionarArea"]/span').click()25# get the total number of classes available (A1, A2, B1, ...)26xpath_class = '//*[@id="form:estrato"]/option'27numClass = browser.find_elements_by_xpath(xpath_class)28numClass = len(numClass)-129# list to store all rows from table30data = []31# begin for class name loop32start = time()33for num in range(numClass):34	# select option: A135	browser.find_element_by_xpath(xpath_class+'['+str(num+2)+']').click()36	className = browser.find_element_by_xpath(xpath_class+'['+str(num+2)+']').text37# left-click to generate the table38	browser.find_element_by_xpath('//*[@id="form:consultar"]').click()39	sleep(1)40	if className == 'A1':41# get web element of the column names42		colsName = browser.find_elements_by_xpath('//*[@id="form"]/div[7]/div/table/thead/tr/th')43# store in labels list the column names44		labels = []45		for name in colsName:46			labels.append(name.text)47# xpath that refers to the whole table structure for each page48	xpath_table = '//*[@id="form"]/div[7]/div/table/tbody/tr'49# xpath that refers to the whole page for data scraping50	xpath_page = '//*[@id="form:j_idt63:j_idt70"]/option'51# get the number of pages for the class52	numPages = browser.find_elements_by_xpath(xpath_page)53	numPages = len(numPages)54	# begin for loop for each page55	for pag in range(numPages):56		print('\nScraping %s at page %d ...' % (className, pag+1))57		browser.find_element_by_xpath(xpath_page+'['+str(pag+1)+']').click()58		sleep(1)59		controlPage = browser.find_element_by_xpath('//*[@id="form:j_idt63:div_paginacao"]/ul/li').text60		controlPage = controlPage.split()61		# get the number of rows for this page62		numRows = int(controlPage[2])-int(controlPage[0])+163		print('Number of rows:', numRows)64		# get every row item and append to the list65		for i in range(numRows):66			issn = browser.find_element_by_xpath(xpath_table+'['+str(i+1)+']/td[1]/span').text67			title = browser.find_element_by_xpath(xpath_table+'['+str(i+1)+']/td[2]').text68			area = browser.find_element_by_xpath(xpath_table+'['+str(i+1)+']/td[3]').text69			classf = browser.find_element_by_xpath(xpath_table+'['+str(i+1)+']/td[4]').text70			data.append((issn, title, area, classf))71# quit the browser72browser.quit()73# print total elapsed time 74print('\nScraping has finished.')75print('Total elapsed scraping time: %.3f min' % ((time()-start)/60) )76# create a dataframe from 'data' list77df = pd.DataFrame.from_records(data, columns=labels)78# print some output info79print('\nDataframe sample:')80print(df.head())81print('\nDataframe dimension: ', df.shape)82print('A .csv file will be generated.')83# write dataframe output csv file...

helpers.py

Source:helpers.py

...7    return "concat('%s')" % s.replace("'", "',\"'\",'")8def contains(tag='*', text=None, prefix='//', class_=None, function='contains'):9    c = ''10    if class_ is not None:11        c = xpath_class(class_)+' and '12    return '''%s%s[%s.//text()[%s(., %s)]]''' % (prefix, tag, c, function, xpath_string_escape(text))13def xpath_class(cls):14    if not cls:15        return '.'16    return '''contains(concat(' ',normalize-space(@class),' '),%s)''' % xpath_string_escape(' '+cls+' ')17class namespace(OrderedDict):18    def __getattr__(self, attr):19        # Make sure we don't mess with OrderedDict's internals (private variables)20        if attr.startswith('_OrderedDict_'):21            return object.__getattr__(self, attr)22        return self[attr]23    def __setattr__(self, attr, data):24        if attr.startswith('_OrderedDict_'):25            return object.__setattr__(self, attr, data)26        self[attr] = data27    def __delattr__(self, attr):...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.