How to use test_progress method in avocado

Best Python code snippet using avocado_python

full_flow_tests.py

Source:full_flow_tests.py

...54def colorize(string, color):55    return color + string + Colors.END56def error(string):57    print(colorize('ERROR: ' + string, Colors.RED))58def test_progress(string):59    print(colorize(string, Colors.BLUE))60def success(string):61    print(colorize(string, Colors.GREEN))62###########63# Checkers64###########65def compare_values(name, expected, actual, rel_tol=5e-2):66    print('Comparing {0}: Expected = {1} ; Actual = {2}'.format(name, expected, actual))67    if math.isclose(expected, actual, rel_tol=rel_tol):68        return True69    else:70        error('Mismatch on {0}'.format(name))71        return False72def accuracy_checker(log, run_dir, expected_results):73    expected_top1, expected_top5 = expected_results74    tops = re.findall(r"Top1: (?P<top1>\d*\.\d*) *Top5: (?P<top5>\d*\.\d*)", log)75    if not tops:76        error('No accuracy results in log')77        return False78    if not compare_values('Top-1', expected_top1, float(tops[-1][0])):79        return False80    return compare_values('Top-5', expected_top5, float(tops[-1][1]))81def earlyexit_accuracy_checker(log, run_dir, expected_results):82    regex_list = (r"Accuracy Stats for exit 0: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",83                  r"Accuracy Stats for exit 1: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",84                  r"Totals for entire network with early exits: top1 = (?P<top1>\d*\.\d*), top5 = (?P<top5>\d*\.\d*)",85                  r"Top1: (?P<top1>\d*\.\d*) *Top5: (?P<top5>\d*\.\d*)")86    for i, regex in enumerate(regex_list):87        if not generic_results_checker(log, regex, expected_results[i]):88            return False89    return True90def generic_results_checker(log, regex1, expected_results):91    actual_results = re.findall(regex1, log)92    if not actual_results:93        error('No results in log')94        return False95    # Grab only the last line of printed results96    actual_results = actual_results[-1]97    # Perform the comparison between expected and actual results98    for (actual_result, expected_result) in zip(actual_results, expected_results):99        if not compare_values('Un-named', expected_result, float(actual_result)):100            return False101    return True102def collateral_checker(log, run_dir, collateral_list):103    """Test that the test produced the expected collaterals.104    A collateral_list is a list of tuples, where tuple elements are:105        0: file name106        1: expected file size107    """108    def relaxed_size_equal(a, b, relaxation):109        return True if abs(a-b) <= relaxation else False110    for collateral in collateral_list:111        file_path = os.path.join(run_dir, collateral[0])112        statinfo = os.stat(file_path)113        if not relaxed_size_equal(statinfo.st_size, collateral[1], 2):114            return False115    return True116###########117# Test Configurations118###########119TestConfig = namedtuple('TestConfig', ['args', 'dataset', 'checker_fn', 'checker_args'])120test_configs = [121    TestConfig('--arch resnet20_cifar_earlyexit --lr=0.3 --epochs=180 --earlyexit_thresholds 0.9 '122               '--earlyexit_lossweights 0.3 --epochs 2 -p 50', DS_CIFAR, earlyexit_accuracy_checker,123               [(99.675, 100.),124                (31.863, 84.985),125                (36.040, 85.910),126                (33.50, 85.980)]),127    TestConfig('--arch simplenet_cifar --epochs 2', DS_CIFAR, accuracy_checker, [47.43, 92.51]),128    TestConfig('-a resnet20_cifar --resume {0} --quantize-eval --evaluate --qe-dynamic --qe-clip-acts avg --qe-no-clip-layers {1}'.129               format(os.path.join(examples_root, 'ssl', 'checkpoints', 'checkpoint_trained_dense.pth.tar'), 'fc'),130               DS_CIFAR, accuracy_checker, [91.47, 99.62]),131    TestConfig('-a preact_resnet20_cifar --epochs 2 --compress {0}'.132               format(os.path.join('full_flow_tests', 'preact_resnet20_cifar_pact_test.yaml')),133               DS_CIFAR, accuracy_checker, [47.47, 93.87]),134    TestConfig('-a resnet20_cifar --resume {0} --sense=filter --sense-range 0 0.10 0.05'.135               format(os.path.join(examples_root, 'ssl', 'checkpoints', 'checkpoint_trained_dense.pth.tar')),136               DS_CIFAR, collateral_checker, [('sensitivity.csv', 3172), ('sensitivity.png', 96157)]),137    TestConfig('--arch simplenet_mnist --epochs 3 -p=50 --compress={0}'.138               format(os.path.join('full_flow_tests', 'simplenet_mnist_pruning.yaml')),139               DS_MNIST, accuracy_checker, [98.82, 100.00]),140]141###########142# Tests Execution143###########144def process_failure(msg, test_idx, cmd, log_path, failed_tests, log):145    error(msg)146    if not log_path:147        test_progress('Log file not created. Full output from test:')148        print(log)149    else:150        test_progress('Test log file: {0}'.format(colorize(log_path, Colors.YELLOW)))151    failed_tests.append((test_idx, cmd, log_path))152def validate_dataset_path(path, default, name):153    if path:154        path = os.path.expanduser(path)155        if not os.path.isdir(path):156            error("Path provided to {0} dataset doesn't exist".format(name))157            exit(1)158        return path159    test_progress('Path to {0} dataset not provided, defaulting to: {1}'.format(name,160                                                                                colorize(os.path.abspath(default),161                                                                                         Colors.WHITE)))162    try:163        os.makedirs(default)164    except OSError as e:165        if e.errno != errno.EEXIST:166            raise167    return default168def run_tests():169    parser = argparse.ArgumentParser()170    parser.add_argument('--cifar10-path', dest='cifar10_path', metavar='DIR', help='Path to CIFAR-10 dataset')171    parser.add_argument('--mnist-path', dest='mnist_path', metavar='DIR', help='Path to MNIST dataset')172    args = parser.parse_args()173    cifar10_path = validate_dataset_path(args.cifar10_path, default='data.cifar10', name='CIFAR-10')174    mnist_path = validate_dataset_path(args.mnist_path, default='data.mnist', name='MNIST')175    datasets = {DS_CIFAR: cifar10_path, DS_MNIST: mnist_path}176    total_configs = len(test_configs)177    failed_tests = []178    for idx, tc in enumerate(test_configs):179        print('')180        test_progress('-------------------------------------------------')181        test_progress('Running Test {0} / {1}'.format(idx + 1, total_configs))182        dataset_dir = datasets[tc.dataset]183        # Run with '--det -j 1' to ensure deterministic results184        # Run with single GPU (lowest common denominator...)185        cmd = 'python3 {script} {tc_args} --det -j 1 --gpus 0 {data}'.format(script=script_path, tc_args=tc.args,186                                                                    data=dataset_dir)187        test_progress('Executing command: ' + colorize(cmd, Colors.YELLOW))188        p = subprocess.Popen(cmd.split(' '), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)189        # Poll for completion190        waiting_chars = ['-', '\\', '|', '/']191        cnt = 0192        while p.poll() is None:193            print(waiting_chars[cnt] * 5, end='\r', flush=True)194            cnt = (cnt + 1) % len(waiting_chars)195            time.sleep(0.5)196        log = p.stdout.read()197        log_path = re.match(r"Log file for this run: (.*)", log)198        log_path = log_path.groups()[0] if log_path else ''199        if p.returncode != 0:200            process_failure('Command returned with exit status {0}'.201                            format(p.returncode), idx, cmd, log_path, failed_tests, log)202            continue203        test_progress('Running checker: ' + colorize(tc.checker_fn.__name__, Colors.YELLOW))204        if not tc.checker_fn(log, os.path.split(log_path)[0], tc.checker_args):205            process_failure('Checker failed', idx, cmd, log_path, failed_tests, log)206            continue207        success('TEST PASSED')208        test_progress('Test log file: {0}'.format(colorize(log_path, Colors.YELLOW)))209    print('')210    test_progress('-------------------------------------------------')211    test_progress('-------------------------------------------------')212    test_progress('All tests completed')213    test_progress('# Tests run: {0} ; # Tests passed {1} ; # Tests failed: {2}'.214                  format(total_configs, total_configs - len(failed_tests), len(failed_tests)))215    if failed_tests:216        print('')217        print(colorize('Failed tests summary:', Colors.RED))218        for idx, cmd, log_path in failed_tests:219            print(colorize('  Test Index:', Colors.YELLOW), idx + 1)220            print(colorize('    Command Line:', Colors.YELLOW), cmd)221            print(colorize('    Log File Path:', Colors.YELLOW), log_path)222        exit(1)223    print('')224    success('ALL TESTS PASSED')225    print('')226    exit(0)227if __name__ == '__main__':...

TestAssistant.py

Source:TestAssistant.py

1from fuzzywuzzy import process as fuzzyprocess2import itertools3class TestProgress:4    def __init__(self,test):5        self.test = test6        self.matched_keyword_groups = []7        self.completed_subtests = []8    def __repr__(self):9        s = ""10        for kwg in self.matched_keyword_groups:11            s+=kwg.description+"->"12        # ??? May be list equality won't work in python13        s+=self.test.result if (self.matched_keyword_groups ==14                self.test.keyword_groups) else " ??? "15        for subtest in self.completed_subtests:16            s+=" -> "+str(subtest)17        return s18class TestAssistant:19    def __init__(self,substitutions_list=None):20        self.test_progress_dict = {}21        self.substitutions_list = substitutions_list22    def has_test_fully_completed(self,test):23        if test not in self.test_progress_dict:24            return False25        test_progress = self.test_progress_dict[test]26        return (len(test_progress.matched_keyword_groups) ==27                len(test.keyword_groups) and28                len(test_progress.completed_subtests) == len(test.subtests))29    def try_match(self,test,user_input_str):30        test_progress = TestProgress(test)31        if test not in self.test_progress_dict:32            self.test_progress_dict[test] = test_progress33        chop_at = 034        for kw in test.keyword_groups[0].keywords:35            """ If the first group of keywords is not found in user_input_str36                test is considered failed """37            kw_i = self._find_kw_in_str(kw,user_input_str)38            if kw_i==-1: return False39            chop_at = max(kw_i+len(kw),chop_at)40        test_progress.matched_keyword_groups+=[test.keyword_groups[0]]41        user_input_str = user_input_str[chop_at:]42        for kwg in test.keyword_groups[1:]:43            if (len(test_progress.matched_keyword_groups)>44                    len(self.test_progress_dict[test].matched_keyword_groups)):45                self.test_progress_dict[test] = test_progress46            for kw in kwg.keywords:47                kw_i = self._find_kw_in_str(kw,user_input_str)48                if kw_i==-1: return test_progress49                chop_at = max(kw_i+len(kw),chop_at)50            test_progress.matched_keyword_groups+=[kwg]51            user_input_str = user_input_str[chop_at:]52        if (len(test_progress.matched_keyword_groups)>53                len(self.test_progress_dict[test].matched_keyword_groups)):54            self.test_progress_dict[test] = test_progress55        for subtest in test.subtests:56            # !!! Possible bug when subtest is partially matched.57            # With current data, it seems like there won't be any problem58            res = self.try_match(subtest,user_input_str)59            if res:60                test_progress.completed_subtests+=[subtest]61            else:62                # !!! If a test with no keywords, but only subtests are63                # introduced, this code will break64                return test_progress65            if (len(test_progress.completed_subtests)>66                    len(self.test_progress_dict[test].completed_subtests)):67                self.test_progress_dict[test] = test_progress68        return True69    def try_continue_match(self,test_progress,user_input_str):70        # test_progress = self.test_progress_dict[test]71        test = test_progress.test72        for kwg in test.keyword_groups:73            if kwg in test_progress.matched_keyword_groups:74                continue75            chop_at = 076            for kw in kwg.keywords:77                kw_i = self._find_kw_in_str(kw,user_input_str)78                if kw_i==-1: return test_progress79                chop_at = max(kw_i+len(kw),chop_at)80            test_progress.matched_keyword_groups+=[kwg]81            user_input_str = user_input_str[chop_at:]82            if (len(test_progress.matched_keyword_groups)>83                    len(self.test_progress_dict[test].matched_keyword_groups)):84                self.test_progress_dict[test] = test_progress85        for subtest in test.subtests:86            # !!! Possible bug when subtest is partially matched.87            # With current data, it seems like there won't be any problem88            res = self.try_match(subtest,user_input_str)89            if res:90                test_progress.completed_subtests+=[subtest]91            else:92                # !!! If a test with no keywords, but only subtests are93                # introduced, this code will break94                return test_progress95            if (len(test_progress.completed_subtests)>96                    len(self.test_progress_dict[test].completed_subtests)):97                self.test_progress_dict[test] = test_progress98        return True99    def _find_kw_in_str(self,kw,string):100        """ Check whether string contains kw, or any variations of kw101            provided by substitutions_list """102        def _n_at_a_time_gen(list,n):103            for i,v in enumerate(list):104                if i < len(list):105                    yield [l for l in list[i:i+n]]106        if len(string)==0:107            return -1108        kw = kw.lower()109        string = string.lower()110        match,rate = fuzzyprocess.extractOne(111                kw, (" ".join(sl) for sl in _n_at_a_time_gen(string.split(),len(kw.split()))112                ))113        # if rate>80:114            # print(" %2f %% %s " % (rate,match))115        if rate>90:116            return string.find(match)117        elif self.substitutions_list is not None:118            # split and substitute for cases like dil. HCl119            orig_kws = kw.split()120            # print("orig-kkws:"+str(orig_kws))121            has_substitutions = list(itertools.filterfalse(122                    lambda kw:kw not in self.substitutions_list,123                    orig_kws))124            # print("has_sub:"+str(has_substitutions))125            if len(has_substitutions)==0 :126                return -1127            substitutions = []128            substitutions+= [ ([kw]+self.substitutions_list[kw]\129                    if kw in self.substitutions_list else [kw])\130                    for kw in orig_kws]131            # print(substitutions)132            # [:1] to exclude first element which is just the original one that was passed133            substitution_combs = list(itertools.product(*substitutions))[1:]134            # print(substitution_combs)135            for sub in substitution_combs:136                i = self._find_kw_in_str(" ".join(sub),string)137                if i!=-1: return i...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.