How to use test_progress method in avocado

Best Python code snippet using avocado_python Github


Full Screen

...54def colorize(string, color):55 return color + string + Colors.END56def error(string):57 print(colorize('ERROR: ' + string, Colors.RED))58def test_progress(string):59 print(colorize(string, Colors.BLUE))60def success(string):61 print(colorize(string, Colors.GREEN))62###########63# Checkers64###########65def compare_values(name, expected, actual, rel_tol=5e-2):66 print('Comparing {0}: Expected = {1} ; Actual = {2}'.format(name, expected, actual))67 if math.isclose(expected, actual, rel_tol=rel_tol):68 return True69 else:70 error('Mismatch on {0}'.format(name))71 return False72def accuracy_checker(log, run_dir, expected_results):73 expected_top1, expected_top5 = expected_results74 tops = re.findall(r"Top1: (?P<top1>\d*\.\d*) *Top5: (?P<top5>\d*\.\d*)", log)75 if not tops:76 error('No accuracy results in log')77 return False78 if not compare_values('Top-1', expected_top1, float(tops[-1][0])):79 return False80 return compare_values('Top-5', expected_top5, float(tops[-1][1]))81def earlyexit_accuracy_checker(log, run_dir, expected_results):82 regex_list = (r"Accuracy Stats for exit 0: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",83 r"Accuracy Stats for exit 1: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",84 r"Totals for entire network with early exits: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",85 r"Top1: (?P<top1>\d*\.\d*) *Top5: (?P<top5>\d*\.\d*)")86 for i, regex in enumerate(regex_list):87 if not generic_results_checker(log, regex, expected_results[i]):88 return False89 return True90def generic_results_checker(log, regex1, expected_results):91 actual_results = re.findall(regex1, log)92 if not actual_results:93 error('No results in log')94 return False95 # Grab only the last line of printed results96 actual_results = actual_results[-1]97 # Perform the comparison between expected and actual results98 for (actual_result, expected_result) in zip(actual_results, expected_results):99 if not compare_values('Un-named', expected_result, float(actual_result)):100 return False101 return True102def collateral_checker(log, run_dir, collateral_list):103 """Test that the test produced the expected collaterals.104 A collateral_list is a list of tuples, where tuple elements are:105 0: file name106 1: expected file size107 """108 def relaxed_size_equal(a, b, relaxation):109 return True if abs(a-b) <= relaxation else False110 for collateral in collateral_list:111 file_path = os.path.join(run_dir, collateral[0])112 statinfo = os.stat(file_path)113 if not relaxed_size_equal(statinfo.st_size, collateral[1], 2):114 return False115 return True116###########117# Test Configurations118###########119TestConfig = namedtuple('TestConfig', ['args', 'dataset', 'checker_fn', 'checker_args'])120test_configs = [121 TestConfig('--arch resnet20_cifar_earlyexit --lr=0.3 --epochs=180 --earlyexit_thresholds 0.9 '122 '--earlyexit_lossweights 0.3 --epochs 2 -p 50', DS_CIFAR, earlyexit_accuracy_checker,123 [(99.675, 100.),124 (31.863, 84.985),125 (36.040, 85.910),126 (33.50, 85.980)]),127 TestConfig('--arch simplenet_cifar --epochs 2', DS_CIFAR, accuracy_checker, [47.43, 92.51]),128 TestConfig('-a resnet20_cifar --resume {0} --quantize-eval --evaluate --qe-dynamic --qe-clip-acts avg --qe-no-clip-layers {1}'.129 format(os.path.join(examples_root, 'ssl', 'checkpoints', 'checkpoint_trained_dense.pth.tar'), 'fc'),130 DS_CIFAR, accuracy_checker, [91.47, 99.62]),131 TestConfig('-a preact_resnet20_cifar --epochs 2 --compress {0}'.132 format(os.path.join('full_flow_tests', 'preact_resnet20_cifar_pact_test.yaml')),133 DS_CIFAR, accuracy_checker, [47.47, 93.87]),134 TestConfig('-a resnet20_cifar --resume {0} --sense=filter --sense-range 0 0.10 0.05'.135 format(os.path.join(examples_root, 'ssl', 'checkpoints', 'checkpoint_trained_dense.pth.tar')),136 DS_CIFAR, collateral_checker, [('sensitivity.csv', 3172), ('sensitivity.png', 96157)]),137 TestConfig('--arch simplenet_mnist --epochs 3 -p=50 --compress={0}'.138 format(os.path.join('full_flow_tests', 'simplenet_mnist_pruning.yaml')),139 DS_MNIST, accuracy_checker, [98.82, 100.00]),140]141###########142# Tests Execution143###########144def process_failure(msg, test_idx, cmd, log_path, failed_tests, log):145 error(msg)146 if not log_path:147 test_progress('Log file not created. Full output from test:')148 print(log)149 else:150 test_progress('Test log file: {0}'.format(colorize(log_path, Colors.YELLOW)))151 failed_tests.append((test_idx, cmd, log_path))152def validate_dataset_path(path, default, name):153 if path:154 path = os.path.expanduser(path)155 if not os.path.isdir(path):156 error("Path provided to {0} dataset doesn't exist".format(name))157 exit(1)158 return path159 test_progress('Path to {0} dataset not provided, defaulting to: {1}'.format(name,160 colorize(os.path.abspath(default),161 Colors.WHITE)))162 try:163 os.makedirs(default)164 except OSError as e:165 if e.errno != errno.EEXIST:166 raise167 return default168def run_tests():169 parser = argparse.ArgumentParser()170 parser.add_argument('--cifar10-path', dest='cifar10_path', metavar='DIR', help='Path to CIFAR-10 dataset')171 parser.add_argument('--mnist-path', dest='mnist_path', metavar='DIR', help='Path to MNIST dataset')172 args = parser.parse_args()173 cifar10_path = validate_dataset_path(args.cifar10_path, default='data.cifar10', name='CIFAR-10')174 mnist_path = validate_dataset_path(args.mnist_path, default='data.mnist', name='MNIST')175 datasets = {DS_CIFAR: cifar10_path, DS_MNIST: mnist_path}176 total_configs = len(test_configs)177 failed_tests = []178 for idx, tc in enumerate(test_configs):179 print('')180 test_progress('-------------------------------------------------')181 test_progress('Running Test {0} / {1}'.format(idx + 1, total_configs))182 dataset_dir = datasets[tc.dataset]183 # Run with '--det -j 1' to ensure deterministic results184 # Run with single GPU (lowest common denominator...)185 cmd = 'python3 {script} {tc_args} --det -j 1 --gpus 0 {data}'.format(script=script_path, tc_args=tc.args,186 data=dataset_dir)187 test_progress('Executing command: ' + colorize(cmd, Colors.YELLOW))188 p = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)189 # Poll for completion190 waiting_chars = ['-', '\\', '|', '/']191 cnt = 0192 while p.poll() is None:193 print(waiting_chars[cnt] * 5, end='\r', flush=True)194 cnt = (cnt + 1) % len(waiting_chars)195 time.sleep(0.5)196 log = log_path = re.match(r"Log file for this run: (.*)", log)198 log_path = log_path.groups()[0] if log_path else ''199 if p.returncode != 0:200 process_failure('Command returned with exit status {0}'.201 format(p.returncode), idx, cmd, log_path, failed_tests, log)202 continue203 test_progress('Running checker: ' + colorize(tc.checker_fn.__name__, Colors.YELLOW))204 if not tc.checker_fn(log, os.path.split(log_path)[0], tc.checker_args):205 process_failure('Checker failed', idx, cmd, log_path, failed_tests, log)206 continue207 success('TEST PASSED')208 test_progress('Test log file: {0}'.format(colorize(log_path, Colors.YELLOW)))209 print('')210 test_progress('-------------------------------------------------')211 test_progress('-------------------------------------------------')212 test_progress('All tests completed')213 test_progress('# Tests run: {0} ; # Tests passed {1} ; # Tests failed: {2}'.214 format(total_configs, total_configs - len(failed_tests), len(failed_tests)))215 if failed_tests:216 print('')217 print(colorize('Failed tests summary:', Colors.RED))218 for idx, cmd, log_path in failed_tests:219 print(colorize(' Test Index:', Colors.YELLOW), idx + 1)220 print(colorize(' Command Line:', Colors.YELLOW), cmd)221 print(colorize(' Log File Path:', Colors.YELLOW), log_path)222 exit(1)223 print('')224 success('ALL TESTS PASSED')225 print('')226 exit(0)227if __name__ == '__main__':...

Full Screen

Full Screen Github


Full Screen

1from fuzzywuzzy import process as fuzzyprocess2import itertools3class TestProgress:4 def __init__(self,test):5 self.test = test6 self.matched_keyword_groups = []7 self.completed_subtests = []8 def __repr__(self):9 s = ""10 for kwg in self.matched_keyword_groups:11 s+=kwg.description+"->"12 # ??? May be list equality won't work in python13 s+=self.test.result if (self.matched_keyword_groups ==14 self.test.keyword_groups) else " ??? "15 for subtest in self.completed_subtests:16 s+=" -> "+str(subtest)17 return s18class TestAssistant:19 def __init__(self,substitutions_list=None):20 self.test_progress_dict = {}21 self.substitutions_list = substitutions_list22 def has_test_fully_completed(self,test):23 if test not in self.test_progress_dict:24 return False25 test_progress = self.test_progress_dict[test]26 return (len(test_progress.matched_keyword_groups) ==27 len(test.keyword_groups) and28 len(test_progress.completed_subtests) == len(test.subtests))29 def try_match(self,test,user_input_str):30 test_progress = TestProgress(test)31 if test not in self.test_progress_dict:32 self.test_progress_dict[test] = test_progress33 chop_at = 034 for kw in test.keyword_groups[0].keywords:35 """ If the first group of keywords is not found in user_input_str36 test is considered failed """37 kw_i = self._find_kw_in_str(kw,user_input_str)38 if kw_i==-1: return False39 chop_at = max(kw_i+len(kw),chop_at)40 test_progress.matched_keyword_groups+=[test.keyword_groups[0]]41 user_input_str = user_input_str[chop_at:]42 for kwg in test.keyword_groups[1:]:43 if (len(test_progress.matched_keyword_groups)>44 len(self.test_progress_dict[test].matched_keyword_groups)):45 self.test_progress_dict[test] = test_progress46 for kw in kwg.keywords:47 kw_i = self._find_kw_in_str(kw,user_input_str)48 if kw_i==-1: return test_progress49 chop_at = max(kw_i+len(kw),chop_at)50 test_progress.matched_keyword_groups+=[kwg]51 user_input_str = user_input_str[chop_at:]52 if (len(test_progress.matched_keyword_groups)>53 len(self.test_progress_dict[test].matched_keyword_groups)):54 self.test_progress_dict[test] = test_progress55 for subtest in test.subtests:56 # !!! Possible bug when subtest is partially matched.57 # With current data, it seems like there won't be any problem58 res = self.try_match(subtest,user_input_str)59 if res:60 test_progress.completed_subtests+=[subtest]61 else:62 # !!! If a test with no keywords, but only subtests are63 # introduced, this code will break64 return test_progress65 if (len(test_progress.completed_subtests)>66 len(self.test_progress_dict[test].completed_subtests)):67 self.test_progress_dict[test] = test_progress68 return True69 def try_continue_match(self,test_progress,user_input_str):70 # test_progress = self.test_progress_dict[test]71 test = test_progress.test72 for kwg in test.keyword_groups:73 if kwg in test_progress.matched_keyword_groups:74 continue75 chop_at = 076 for kw in kwg.keywords:77 kw_i = self._find_kw_in_str(kw,user_input_str)78 if kw_i==-1: return test_progress79 chop_at = max(kw_i+len(kw),chop_at)80 test_progress.matched_keyword_groups+=[kwg]81 user_input_str = user_input_str[chop_at:]82 if (len(test_progress.matched_keyword_groups)>83 len(self.test_progress_dict[test].matched_keyword_groups)):84 self.test_progress_dict[test] = test_progress85 for subtest in test.subtests:86 # !!! Possible bug when subtest is partially matched.87 # With current data, it seems like there won't be any problem88 res = self.try_match(subtest,user_input_str)89 if res:90 test_progress.completed_subtests+=[subtest]91 else:92 # !!! If a test with no keywords, but only subtests are93 # introduced, this code will break94 return test_progress95 if (len(test_progress.completed_subtests)>96 len(self.test_progress_dict[test].completed_subtests)):97 self.test_progress_dict[test] = test_progress98 return True99 def _find_kw_in_str(self,kw,string):100 """ Check whether string contains kw, or any variations of kw101 provided by substitutions_list """102 def _n_at_a_time_gen(list,n):103 for i,v in enumerate(list):104 if i < len(list):105 yield [l for l in list[i:i+n]]106 if len(string)==0:107 return -1108 kw = kw.lower()109 string = string.lower()110 match,rate = fuzzyprocess.extractOne(111 kw, (" ".join(sl) for sl in _n_at_a_time_gen(string.split(),len(kw.split()))112 ))113 # if rate>80:114 # print(" %2f %% %s " % (rate,match))115 if rate>90:116 return string.find(match)117 elif self.substitutions_list is not None:118 # split and substitute for cases like dil. HCl119 orig_kws = kw.split()120 # print("orig-kkws:"+str(orig_kws))121 has_substitutions = list(itertools.filterfalse(122 lambda kw:kw not in self.substitutions_list,123 orig_kws))124 # print("has_sub:"+str(has_substitutions))125 if len(has_substitutions)==0 :126 return -1127 substitutions = []128 substitutions+= [ ([kw]+self.substitutions_list[kw]\129 if kw in self.substitutions_list else [kw])\130 for kw in orig_kws]131 # print(substitutions)132 # [:1] to exclude first element which is just the original one that was passed133 substitution_combs = list(itertools.product(*substitutions))[1:]134 # print(substitution_combs)135 for sub in substitution_combs:136 i = self._find_kw_in_str(" ".join(sub),string)137 if i!=-1: return i...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:


You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run avocado automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?