How to use get_subdir_list method in autotest

Best Python code snippet using autotest_python Github


Full Screen

...22plt.rc('axes', titlesize=label_size)23plt.rc('axes', labelsize=label_size)24plt.rc('xtick', labelsize=tick_size)25plt.rc('ytick', labelsize=tick_size)26def get_subdir_list(path_to_root_dir: str):27 # search_string = pathlib.Path(path_to_root_dir) / '*'28 # subdir_list = glob.glob(str(search_string), recursive=False)29 p = pathlib.Path(path_to_root_dir)30 subdir_list = [d for d in p.iterdir() if d.is_dir()]31 return subdir_list32def merge_splitted_results_from_subdir(subdir_list: list):33 """34 Extract all adversarials.t from path sub directories, assert that the parameters are the same and unify them35 :param subdir_list: The path to the directory containing subdir that should be unified.36 :return: A unified adversarials object.37 """38 assert(len(subdir_list) > 1)39 params1 = load_dir_params(subdir_list[0])40 # Make sure all subdir contains the same configuration besides the indices:41 for i, subdir in enumerate(subdir_list[1:]):42 params2 = load_dir_params(subdir)43 assert(is_dicts_equal(params1["adv_attack_test"], params2["adv_attack_test"], ["test_start_idx", "test_end_idx"]))44 assert(is_dicts_equal(params1["model"], params2["model"]))45 assert(is_dicts_equal(params1["fit_to_sample"], params2["fit_to_sample"]))46 adv_l = []47 for i, subdir in enumerate(subdir_list):48 adv_l.append(torch.load(os.path.join(subdir, 'adversarials.t')))49 return adv_l[0].cat(adv_l), params150def is_dicts_equal(d1: dict, d2: dict, ignore_keys: Union[None, list] = None):51 if ignore_keys is None:52 ignore_keys = []53 return {k: v for k, v in d1.items() if k not in ignore_keys} == {k: v for k, v in d2.items() if k not in ignore_keys}54def load_dir_params(path: str):55 param_path = pathlib.Path(path) / 'params.json'56 if param_path.exists() is False:57 param_path = '..' / pathlib.Path(path) / 'params.json'58 param_path_l = glob.glob(str(param_path), recursive=False)59 assert len(param_path_l) == 1, "for param_path: {} found the following param files: {}".format(param_path, param_path_l)60 with open(param_path_l[0]) as f:61 params = json.load(f)62 if params["adv_attack_test"]["white_box"] is False:63 path_to_blackbox_dir = os.path.dirname(params["adv_attack_test"]["black_box_adv_path"])64 attack_params = load_dir_params(path_to_blackbox_dir)65 params["adv_attack_test"] = attack_params["adv_attack_test"]66 assert (params["adv_attack_test"]["white_box"])67 return params68def load_exp_result_from_dir(root_dir: str, indices: Union[list, None] = None):69 """70 Load adversarials results and param dict of an experiment. Results are contained in the root dir or inside multiple71 subdir.72 :param root_dir: The path to the root directory containing the results73 :param indices: if not None, include only results of indices74 :return: adversarial object and a param dict75 """76 subdir_list = get_subdir_list(root_dir)77 if(len(subdir_list) > 1):78 adv, params = merge_splitted_results_from_subdir(subdir_list)79 else:80 if len(subdir_list) == 1:81 root_dir = subdir_list[0]82 try:83 adv = torch.load(os.path.join(root_dir, 'adversarials.t'))84 except FileNotFoundError:85 adv = torch.load(os.path.join(root_dir, 'adversarials_compress.t'))86 params = load_dir_params(root_dir)87 if indices is not None:88 adv.correct = adv.correct[indices]89 adv.loss = adv.loss[indices]90 return adv, params91def compress_and_save_adv(subdir_list: list):92 """93 Save compressed adversarial object, without image data.94 :param subdir_list: a list of subdirectories for which the function will save a compressed version of adversarials.t95 """96 for i, subdir in enumerate(subdir_list):97 adv, params = load_exp_result_from_dir(subdir, None)98 if adv.original_sample is not None or adv.adversarial_sample is not None:99 print("Compress adversarial.t in directory: {}".format(subdir))100 adv.original_sample = None101 adv.adversarial_sample = None102 adv.dump(subdir, "adversarials_compress.t")103def results_dirs_to_df(subdir_list: list, indices: Union[list, None] = None) -> pd.DataFrame:104 """105 Load multiple experiments that are stored in subdir_list and save them into a DF.106 :param subdir_list: a list of folders containing the results107 :param indices: if not None, include only results of indices108 :return: A dataframe containing the results.109 """110 assert(len(subdir_list) >= 1)111 statistics_df = pd.DataFrame(columns=['acc', 'epsilon', 'mean loss', 'samples'])112 for i, subdir in enumerate(subdir_list):113 # params = load_dir_params(subdir)114 # if params["adv_attack_test"]["white_box"] is False:115 # path_to_blackbox_dir = os.path.dirname(params["adv_attack_test"]["black_box_adv_path"])116 # attack_params = load_dir_params(path_to_blackbox_dir)117 # params["adv_attack_test"] = attack_params["adv_attack_test"]118 # assert(params["adv_attack_test"]["white_box"])119 adv, params = load_exp_result_from_dir(subdir, indices)120 statistics_df.loc[i, "acc"] = adv.get_accuracy()121 statistics_df.loc[i, "mean loss"] = adv.get_mean_loss()122 statistics_df.loc[i, "mean size"] = adv.get_mean_loss()123 statistics_df.loc[i, "samples"] = len(adv.correct)124 if params["adv_attack_test"]['attack_type'] == 'natural':125 statistics_df.loc[i, "epsilon"] = 0126 statistics_df.loc[i, "iter"] = 0127 statistics_df.loc[i, "pgd_step"] = 0128 statistics_df.loc[i, "restarts"] = 0129 else:130 statistics_df.loc[i, "epsilon"] = params["adv_attack_test"]["epsilon"]131 statistics_df.loc[i, "iter"] = params["adv_attack_test"]["pgd_iter"]132 statistics_df.loc[i, "pgd_step"] = params["adv_attack_test"]["pgd_step"]133 statistics_df.loc[i, "restarts"] = params["adv_attack_test"]["pgd_test_restart_num"]134 statistics_df.loc[i, "lambda"] = params["fit_to_sample"]["epsilon"]135 return statistics_df136if __name__ == "__main__":137 print("start")138 # All tests were performed on the imagenet evaluation subset which includes all the samples for the first 100 classes (50 samples per each class for a total of 5000 samples).139 #140 # Experiments:141 # 1. pNML Accuracy Vs. Refinement strength (\lambda) - Shows the trade-off between natural and robust accuracy142 # 2. Accuracy Vs. PGD-attack strength (\epsilon); with\ without pNML - Shows that pNML improve accuracy.143 # 3. Accuracy Vs. Adaptive-attack & PGD-attack strength (\epsilon) - Shows that adaptive attack is not working144 # 4. TODO: blackbox attack Accuracy Vs. Blackbox-attack for different epsilons strength (\epsilon) - Shows that pNML improve accuracy compared to blackbox without pNML145 # PGD attack properties: "pgd_iter": 50, "pgd_step": 0.00392156862, "pgd_test_restart_num": 10146 # For 1. "epsilon": 0.0156862745098 was selected.147 # For 2. lambda = 0.01176 (refinement strength)148 # For 3. "epsilon": [0.0156, 0.0313, 0.0627] was selected.; lambda = 0.01176(refinement strength). indices: 1 per label149 # 1.1. pNML Accuracy Vs. Refinement strength (\lambda)150 natural_acc_vs_lambda_res_path = "./output/imagenet_diff_fix_natural"151 pgd_acc_vs_lambda_res_path = "./output/imagenet_diff_fix_pgd"152 # 1.2. Accuracy Vs. attack strength (\epsilon); with\ without pNML153 pgd_nopnml_diff_eps_path = "./output/imagenet_pgd_diff_eps"154 pgd_pnml_diff_eps_path = "./output/imagenet_pgd_diff_eps_pnml_lambda_01176"155 # 1.3 Accuracy Vs. Adaptive-attack & PGD-attack strength (\epsilon)156 adaptive_diff_eps_path = "./output/imagenet_adaptive"157 # 1.1158 subdir_list = get_subdir_list(pgd_acc_vs_lambda_res_path)159 df_pgd = results_dirs_to_df(subdir_list)160 subdir_list = get_subdir_list(natural_acc_vs_lambda_res_path)161 df_natural = results_dirs_to_df(subdir_list)162 df_natural = df_natural.sort_values("lambda", ignore_index=True)163 df_pgd = df_pgd.sort_values("lambda", ignore_index=True)164 axes = plt.gca()165 # axes.set_xlim([0.0, 0.12])166 # axes.set_ylim([0.0, 1.0])167 plt.figure(0)168 plt.plot(df_natural['lambda'], df_natural['acc'], 'go-', label='Natural')169 plt.plot(df_pgd['lambda'], df_pgd['acc'], 'bo-', label='PGD')170 plt.legend(fontsize=12, ncol=1, loc=1)171 plt.xlabel('Refinement strength')172 plt.ylabel('Accuracy')173 plt.savefig(os.path.join(output_path, 'Natural_vs_refinement_strength.jpg'), dpi=200,174 bbox_inches=plt.tight_layout()) if is_save_fig else None175 # 1.2177 subdir_list = get_subdir_list(pgd_nopnml_diff_eps_path)178 df_pgd_nopnml = results_dirs_to_df(subdir_list)179 subdir_list = get_subdir_list(pgd_pnml_diff_eps_path)180 df_pgd_pnml = results_dirs_to_df(subdir_list)181 df_pgd_nopnml = df_pgd_nopnml.sort_values("epsilon", ignore_index=True)182 df_pgd_pnml = df_pgd_pnml.sort_values("epsilon", ignore_index=True)183 axes = plt.gca()184 # axes.set_xlim([0.0, 0.12])185 # axes.set_ylim([0.0, 1.0])186 plt.figure(1)187 plt.plot(df_pgd_nopnml['epsilon'], df_pgd_nopnml['acc'], 'go--', label='no pNML')188 plt.plot(df_pgd_pnml['epsilon'], df_pgd_pnml['acc'], 'bo--', label='pNML')189 plt.legend(fontsize=12, ncol=1, loc=1)190 plt.xlabel('Attack strength')191 plt.ylabel('Accuracy')192 plt.savefig(os.path.join(output_path, 'pgd_pnml_and_nopnml.jpg'), dpi=200,193 bbox_inches=plt.tight_layout()) if is_save_fig else None194 # 1.3196 indices = [i for i in range(0, 4999+1, 50)]197 subdir_list = get_subdir_list(adaptive_diff_eps_path)198 df_adaptive = results_dirs_to_df(subdir_list)199 df_adaptive = df_adaptive.sort_values("epsilon", ignore_index=True)200 subdir_list = get_subdir_list(pgd_pnml_diff_eps_path)201 df_pgd_pnml = results_dirs_to_df(subdir_list, indices)202 axes = plt.gca()203 plt.figure(2)204 plt.plot(df_adaptive['epsilon'], df_adaptive['acc'], 'go--', label='Adaptive')205 plt.plot(df_pgd_pnml['epsilon'], df_pgd_pnml['acc'], 'bo--', label='PGD')206 plt.legend(fontsize=12, ncol=1, loc=1)207 plt.xlabel('Attack strength')208 plt.ylabel('Accuracy')209 plt.savefig(os.path.join(output_path, 'adaptive_vs_pgd.jpg'), dpi=200,210 bbox_inches=plt.tight_layout()) if is_save_fig else None...

Full Screen

Full Screen Github


Full Screen

...21 22 return '\n'.join(items)23 24def get_qrc_xml(topdir):25 subdirs = get_subdir_list(topdir)26 qresources = ["<RCC>"]27 for _dir in subdirs:28 qresources.append(get_qresource_xml_bydir(topdir, _dir))29 qresources.append("</RCC>")30 31 return '\n'.join(qresources)32def main(topdir, qrcfile):33 rcc_xml = get_qrc_xml(topdir)34 open("tmp.qrc", 'w').write(rcc_xml)35 return system("pyrcc4 -py3 tmp.qrc > %s" % qrcfile)36 37if __name__ == "__main__":38 argc = len(sys.argv)39 if 1 < argc < 3:...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:


You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run autotest automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?