Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More
How to use get_test_labels method in autotest
Best Python code snippet using autotest_python
experiment.py
Source:experiment.py 
...120121 return np.array(labels, np.int8)122123124def get_test_labels():125 labels = []126 with open(path.TEST_RESULT_TXT, "r") as f:127 for i in f.readlines():128 result = i.strip().split(",")[1:]129 result = [int(c) for c in result]130 labels.append(result)131132 return np.array(labels, np.int8)133134135def build_epoch_test(y_true, y_pred, weight_file, identifier=None):136 all_f2 = get_epoch_test()137 if not identifier:138 identifier = get_epoch_identifier(weight_file)139140 if all_f2.get(identifier):141 print(f"prediction is evaluated, {identifier}")142 return143 one_label_greedy_f2_all = []144 for i in range(13):145 one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))146147 f2 = {"avg": np.mean(one_label_greedy_f2_all)}148 for i in range(13):149 f2[f"{i}"] = one_label_greedy_f2_all[i]150151 all_f2[identifier] = f2152 save_epcoh_test(all_f2)153154155def get_ablation_experiment_predict(mode_path, val):156 y_true = get_test_labels()157158 original_test_file = []159 with open(path.TEST_DATA_TXT, 'r') as f:160 for i in f.readlines():161 image_name = i.split(",")[0] + ".jpg"162 original_test_file.append(os.path.join(path.ORIGINAL_TEST_IMAGES_PATH, image_name))163164 weight_files = []165 predict_files = []166 _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)167168 for root, dirs, files in os.walk(mode_path):169 for file in files:170 if "hdf5(test)" in file:171 predict_files.append(os.path.join(root, file))172 continue173 if not file.split(".")[-1] == "hdf5":174 continue175 if f"val{val}" not in root:176 continue177178 model_num = re.match(r".*model([0-9]*).*", root).group(1)179 if int(model_num) < 100:180 continue181182 weight_files.append(os.path.join(root, file))183184 for predict_file in predict_files:185 print(f"evaluate {predict_file}")186 weight_file = predict_file.replace("(test).predict.npy", "")187 y_pred = np.load(predict_file)188 build_epoch_test(y_true, y_pred, weight_file)189190 for weight_file in weight_files:191 print(f"evaluate {weight_file}")192 unique_path = re.match(r".*competition[\\/]*(.*)", weight_file).group(1)193 identifier = "-".join(unique_path.split("\\"))194 print(f"id {identifier}")195 cnn_result_path = os.path.join(weight_file + "(test)")196 print(f"result {cnn_result_path}")197 if not os.path.exists(keras_util.get_prediction_path(cnn_result_path)):198 print(cnn_result_path)199 attr_get_model, attr_model_config = keras_util.dynamic_model_import(weight_file)200 model = attr_get_model(output_dim=len(attr_model_config.label_position), weights=None)201 model.load_weights(weight_file)202 attr_model_config.val_files = []203 for data_type in attr_model_config.data_type:204 if data_type == path.DATA_TYPE_ORIGINAL:205 attr_model_config.val_files.append(original_test_file)206207 attr_model_config.tta_flip = True208 y_pred = keras_util.predict_tta(model, attr_model_config, verbose=1)209210 for i in range(13):211 y_pred[:, i] = y_pred[:, i] > thresholds[i][weight_file]212213 y_pred = y_pred.astype(np.int8)214215 keras_util.save_prediction_file(y_pred, cnn_result_path)216 else:217 y_pred = np.load(keras_util.get_prediction_path(cnn_result_path))218219 build_epoch_test(y_true, y_pred, weight_file)220221222def calc_xgb_f2_score():223 y_true = get_test_labels()224 y_pred = get_xgb_result()225226 with open(os.path.join(path.RESULT_PATH, "test.txt"), "a+") as f:227 one_label_greedy_f2_all = []228229 for i in range(13):230 one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))231232 f.write("\n\n")233 f.write("Greedy F2-Score is: %f\n" % np.average(one_label_greedy_f2_all))234 for i in range(13):235 f.write("[label %d] greedy-f2=%4f\n" % (i, one_label_greedy_f2_all[i]))236237238def get_existed_cnn_f2_score(val, mode_path):239 y_true = get_test_labels()240241 original_test_file = []242 cnt = 0243 with open(path.TEST_DATA_TXT, 'r') as f:244 for i in f.readlines():245 image_name = i.split(",")[0] + ".jpg"246 original_test_file.append(os.path.join(path.ORIGINAL_TEST_IMAGES_PATH, image_name))247248 weight_files = []249 _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)250251 for root, dirs, files in os.walk(mode_path):252 for file in files:253 if "predict.np" not in file or f"val{val}" not in root:254 continue255 weight_files.append(os.path.join(root, file)[:-12])256257 result = [{} for i in range(13)]258 average = {}259260 with open(os.path.join(path.RESULT_PATH, f"test(training)(all)(val{val}).txt"), "w+") as f:261 for weight_file in weight_files:262 print(f"weight file {weight_file}")263 unique_path = re.match(r".*competition[\\/]*(.*)", weight_file).group(1)264 identifier = "-".join(unique_path.split("\\"))265 print(f"id {identifier}")266 cnn_result_path = os.path.join(path.CNN_RESULT_PATH, identifier)267 print(f"result {cnn_result_path}")268 if os.path.exists(keras_util.get_prediction_path(cnn_result_path)):269 y_pred = np.load(keras_util.get_prediction_path(cnn_result_path))270 else:271 print("model not existed")272 continue273274 if thresholds[0].get(weight_file) is None:275 print("threshold not existed")276 continue277278 for i in range(13):279 y_pred[:, i] = y_pred[:, i] > thresholds[i][weight_file]280281 y_pred = y_pred.astype(np.int8)282283 one_label_greedy_f2_all = []284285 for i in range(13):286 # f2, _ = metrics.greedy_f2_score(y_true[:, i], y_pred[:, i], 1)287 f2 = fbeta_score(y_true[:, i], y_pred[:, i], beta=2)288 one_label_greedy_f2_all.append(f2)289 result[i][weight_file] = f2290 average[weight_file] = np.mean(one_label_greedy_f2_all)291292 f.write("\n\n")293 f.write("Weight: %s\n" % cnn_result_path)294 f.write("Greedy F2-Score is: %f\n" % np.mean(one_label_greedy_f2_all))295 for i in range(13):296 f.write("[label %d] greedy-f2=%4f\n" % (i, one_label_greedy_f2_all[i]))297298 print(f"need predict {cnt} model")299300 with open(os.path.join(path.RESULT_PATH, f"test(training)(label)(val{val}).txt"), "w+") as f:301 average = sorted(average.items(), key=lambda x: x[1], reverse=True)302 f.write(f"\n\n==================== all =======================\n\n")303 for item in average:304 f.write("%4f: %s \n" % (item[1], item[0]))305306 for i in range(13):307 dic = result[i]308 dic = sorted(dic.items(), key=lambda x: x[1], reverse=True)309 f.write(f"\n\n====================label {i} =======================\n\n")310 for item in dic:311 f.write("%4f: %s \n" % (item[1], item[0]))312313314def build_epoch_cv(val):315 all_label, one_label, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)316 all_f2 = get_epoch_cv()317318 for all in all_label:319 f2 = {}320 f2["avg"] = all[1]321 all_f2[get_epoch_identifier(all[0])] = f2322323 for label in range(13):324 for one in one_label[label]:325 identifier = get_epoch_identifier(one[0])326 all_f2[identifier][f"{label}"] = one[1]327328 save_epoch_cv(all_f2)329330331def build_model_cv(val):332 all_label, one_label, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)333 all_f2 = get_model_cv()334335 for all in all_label:336 f2 = {}337 f2["avg"] = all[1]338 for i in range(13):339 f2[f"{i}"] = 0340 all_f2[get_model_identifier(all[0])] = f2341342 for label in range(13):343 for one in one_label[label]:344 identifier = get_model_identifier(one[0])345 if one[1] > all_f2[identifier][f"{label}"]:346 all_f2[identifier][f"{label}"] = one[1]347 all_f2[identifier][f"model{label}"] = get_epoch_identifier(one[0])348349 for key in all_f2.keys():350 avg = 0351 for i in range(13):352 avg += all_f2[key][f"{i}"]353 avg /= 13354 all_f2[key]["avg"] = avg355356 save_model_cv(all_f2)357358359def build_model_test():360 epoch_test = get_epoch_test()361 model_cv = get_model_cv()362 model_test = {}363 for id in epoch_test.keys():364 model_test[id.split(".")[0]] = {"avg": 0}365 for i in range(13):366 model_test[id.split(".")[0]][f"{i}"] = 0367368 for id in model_cv:369 value = model_cv[id]370 avg = 0371 for i in range(13):372 model = value[f"model{i}"]373 if epoch_test.get(model):374 test_model = epoch_test[model]375 test_f2 = test_model[f"{i}"]376 model_test[id.split(".")[0]][f"{i}"] = test_f2377 model_test[id.split(".")[0]][f"model{i}"] = model378 avg += test_f2379 model_test[id.split(".")[0]]["avg"] = avg / 13380381 model_test_sorted = sorted(model_test.items(), key=lambda x: x[1]["avg"], reverse=True)382 model_test = {}383 for item in model_test_sorted:384 model_test[item[0]] = item[1]385386 save_model_test(model_test)387388 print("ok")389390391def build_threshold_cv():392 threshold_cv = {}393394 for val in range(1, 6):395 _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)396 print("ok")397398 for label in range(13):399 for weight_path in thresholds[label].keys():400 if not threshold_cv.get(get_epoch_identifier(weight_path)):401 threshold_cv[get_epoch_identifier(weight_path)] = {}402 threshold_cv[get_epoch_identifier(weight_path)][f"{label}"] = thresholds[label][weight_path]403 print("ok")404405 save_threshold_cv(threshold_cv)406407def build_global_cv():408 epoch_cv = get_epoch_cv()409 global_cv = {"avg": 0}410 for i in range(13):411 global_cv[f"{i}"] = 0412413 for id in epoch_cv.keys():414 model = epoch_cv[id]415 for label in range(13):416 if model[f"{label}"] > global_cv[f"{label}"]:417 global_cv[f"{label}"] = model[f"{label}"]418 global_cv[f"model{label}"] = id419420 avg = 0421 for i in range(13):422 avg += global_cv[f"{i}"]423 global_cv["avg"] = avg / 13424 save_global_cv(global_cv)425426427def build_global_test():428 global_cv = get_global_cv()429 epoch_test = get_epoch_test()430 global_test = {"avg": 0}431 for i in range(13):432 global_test[f"{i}"] = 0433434 for label in range(13):435 cv_model = global_cv[f"model{label}"]436 test_model = epoch_test[cv_model]437 f2 = test_model[f"{label}"]438 global_test[f"{label}"] = f2439440 avg = 0441 for i in range(13):442 avg += global_test[f"{i}"]443 global_test["avg"] = avg / 13444445 save_global_test(global_test)446447448def cnn_result_name_to_epoch_name(cnn: str):449 cnn = cnn[:-12]450 cnn = cnn.replace("-", "\\")451 return "\\" + cnn452453454def build_ensemble_epoch_cv():455 result_paths = []456 result_files = {}457458 threshold_cv = get_threshold_cv()459460 for root, dirs, files in os.walk(path.CNN_RESULT_PATH):461 for file in files:462 if "val2" not in file:463 continue464 if file.split(".")[-1] != "npy":465 continue466 result_paths.append(os.path.join(root, file))467 result_files[os.path.join(root, file)] = file468469 for result_path in result_paths:470 file_name = result_files[result_path]471 epoch_name = cnn_result_name_to_epoch_name(file_name)472 y_true = get_test_labels()473 y_pred = np.load(result_path)474475 threshold = threshold_cv[epoch_name]476477 for i in range(13):478 y_pred[:, i] = y_pred[:, i] > threshold[f"{i}"]479480 y_pred = y_pred.astype(np.int8)481482 build_epoch_test(y_true, y_pred, None, identifier=epoch_name)483484485def build_threshold_test():486 result_paths = []487 result_files = {}488489 threshold_test = {}490491 for root, dirs, files in os.walk(path.CNN_RESULT_PATH):492 for file in files:493 if "val2" not in file:494 continue495 if file.split(".")[-1] != "npy":496 continue497 result_paths.append(os.path.join(root, file))498 result_files[os.path.join(root, file)] = file499500 for result_path in result_paths:501 file_name = result_files[result_path]502 epoch_name = cnn_result_name_to_epoch_name(file_name)503504 if threshold_test.get(epoch_name):505 continue506 print(f"epoch name {epoch_name}")507 y_true = get_test_labels()508 y_pred = np.load(result_path)509 threshold_test[epoch_name] = {}510 for i in range(13):511 score, threshold = metrics.greedy_f2_score(y_true[:, i], y_pred[:, i], 1, step=100)512 threshold_test[epoch_name][f"{i}"] = threshold[0]513514 save_threshold_test(threshold_test)515516517def build_epoch_test_standard():518 result_paths = []519 result_files = {}520521 threshold_test = get_threshold_test()522 epoch_test_standard = {}523524 for root, dirs, files in os.walk(path.CNN_RESULT_PATH):525 for file in files:526 if "val2" not in file:527 continue528 if file.split(".")[-1] != "npy":529 continue530 result_paths.append(os.path.join(root, file))531 result_files[os.path.join(root, file)] = file532533 for result_path in result_paths:534 file_name = result_files[result_path]535 epoch_name = cnn_result_name_to_epoch_name(file_name)536 threshold = threshold_test[epoch_name]537 y_true = get_test_labels()538 y_pred = np.load(result_path)539540 for i in range(13):541 y_pred[:, i] = y_pred[:, i] > threshold[f"{i}"]542543 y_pred = y_pred.astype(np.int8)544545 one_label_greedy_f2_all = []546 for i in range(13):547 one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))548549 f2 = {"avg": np.mean(one_label_greedy_f2_all)}550 for i in range(13):551 f2[f"{i}"] = one_label_greedy_f2_all[i]
...
model_analytics.py
Source:model_analytics.py
...14 def get_model(self):15 return self.model16 def get_predictions(self):17 return self.predictions18 def get_test_labels(self):19 return self.test_labels20 def get_feature_list(self):21 return self.feature_list22 def get_feat_imp(self):23 return self.rf_feat_imp24 def save_model(self,file_name):25 try:26 pickle.dump(self.get_model(),open(file_name + ".pickle",'wb'))27 print ("Model successfully saved.")28 except Exception as e:29 print ("Model not saved", e)30 def model_absolute_errors(self):31 # Calculate the absolute errors32 return abs(self.get_predictions() - self.get_test_labels())33 def model_mean_absolute_error(self):34 # Return the mean absolute error (mae)35 return round(np.mean(self.model_absolute_errors()), 4)36 def model_mean_absolute_percentage_error(self):37 # Calculate mean absolute percentage error (MAPE)38 return 100 * (self.model_absolute_errors() / self.get_test_labels())39 def model_accuracy(self):40 # Calculate and display accuracy41 return 100 - np.mean(self.model_absolute_errors())42 def model_importances(self):43 return list(self.get_feat_imp())44 def model_feature_importance(self):45 # List of tuples with variable and importance46 imp_tmp = self.model_importances()47 feat_list_tmp = self.get_feature_list()48 feature_importance = [(feature, round(imp_tmp, 2)) for feature, imp_tmp in zip(feat_list_tmp, imp_tmp)]49 # Sort the feature importances by most important first50 feature_importance = sorted(feature_importance, key=lambda x: x[1], reverse=True)51 # Print out the feature and importances52 [print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importance]...
utils.py
Source:utils.py
1import os2import cv23import glob4import random5import numpy as np6from itertools import chain7def read_image(img_path, is_resize=True, resize_dim=64):8 """9 This function will read each individual image. The image will be converted to gray scale. Finally if specified,10 the image will be resized and returned11 :param img_path: the path of the image to be read12 :param is_resize: Weather to resize the image13 :param resize_dim: The height and width of the resized image14 :return: resized_img15 """16 img = cv2.imread(img_path)17 img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)18 if is_resize:19 img = cv2.resize(img, (resize_dim, resize_dim))20 return img21def read_directory(img_paths, str_labels, is_test=False):22 """23 This function will return the numpy images data and the corresponding labels for the image paths provided.24 :param img_paths: the image paths to be read to images25 :param str_labels:26 :param is_test: defines if we are running code in test or train mode27 :return: classes, resized_img_data28 """29 img_data = np.array([read_image(x) for x in img_paths], dtype=np.uint8)30 if is_test:31 return img_data, None32 labels = np.array([str_labels.index(x.split('/')[-1].split('\\')[0]) for x in img_paths], dtype=np.int)33 return img_data, labels34def get_clf_data(path, train_per=0.8, is_split=True, get_test_labels=True):35 """36 This function will read the entire data from the specified directory. The function will return the image data and37 their corresponding classes.38 :param path: The root directory path of the images39 :param train_per: if split, the percentage of images to be put into train40 :param is_split: defines if data is to be split in train and test41 :param get_test_labels: whether test labels are to be returned.42 :return:43 """44 str_labels = os.listdir(path)45 img_paths = [*chain(*[glob.glob(path + '/{}/*.png'.format(x)) for x in str_labels])]46 random.shuffle(img_paths)47 if is_split:48 n_imgs = len(img_paths)49 n_train = int(train_per * n_imgs)50 train_path, test_path = img_paths[:n_train], img_paths[n_train:]51 train_data, train_labels = read_directory(train_path, str_labels)52 test_data, test_labels = read_directory(test_path, str_labels, is_test=(not get_test_labels))53 np.save('data/str_train_labels.npy', str_labels)54 return train_data, test_data, train_labels, test_labels55 else:56 return read_directory(img_paths, str_labels, is_test=(not get_test_labels))57if __name__ == '__main__':58 data_path = '../../data/fish_dataset/data'59 tr_data, te_data, tr_labels, te_labels = get_clf_data(data_path)...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run autotest automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
