Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use get_test_labels method in autotest

Best Python code snippet using autotest_python

experiment.py

Source:experiment.py

...120121    return np.array(labels, np.int8)122123124def get_test_labels():125    labels = []126    with open(path.TEST_RESULT_TXT, "r") as f:127        for i in f.readlines():128            result = i.strip().split(",")[1:]129            result = [int(c) for c in result]130            labels.append(result)131132    return np.array(labels, np.int8)133134135def build_epoch_test(y_true, y_pred, weight_file, identifier=None):136    all_f2 = get_epoch_test()137    if not identifier:138        identifier = get_epoch_identifier(weight_file)139140    if all_f2.get(identifier):141        print(f"prediction is evaluated, {identifier}")142        return143    one_label_greedy_f2_all = []144    for i in range(13):145        one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))146147    f2 = {"avg": np.mean(one_label_greedy_f2_all)}148    for i in range(13):149        f2[f"{i}"] = one_label_greedy_f2_all[i]150151    all_f2[identifier] = f2152    save_epcoh_test(all_f2)153154155def get_ablation_experiment_predict(mode_path, val):156    y_true = get_test_labels()157158    original_test_file = []159    with open(path.TEST_DATA_TXT, 'r') as f:160        for i in f.readlines():161            image_name = i.split(",")[0] + ".jpg"162            original_test_file.append(os.path.join(path.ORIGINAL_TEST_IMAGES_PATH, image_name))163164    weight_files = []165    predict_files = []166    _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)167168    for root, dirs, files in os.walk(mode_path):169        for file in files:170            if "hdf5(test)" in file:171                predict_files.append(os.path.join(root, file))172                continue173            if not file.split(".")[-1] == "hdf5":174                continue175            if f"val{val}" not in root:176                continue177178            model_num = re.match(r".*model([0-9]*).*", root).group(1)179            if int(model_num) < 100:180                continue181182            weight_files.append(os.path.join(root, file))183184    for predict_file in predict_files:185        print(f"evaluate {predict_file}")186        weight_file = predict_file.replace("(test).predict.npy", "")187        y_pred = np.load(predict_file)188        build_epoch_test(y_true, y_pred, weight_file)189190    for weight_file in weight_files:191        print(f"evaluate {weight_file}")192        unique_path = re.match(r".*competition[\\/]*(.*)", weight_file).group(1)193        identifier = "-".join(unique_path.split("\\"))194        print(f"id {identifier}")195        cnn_result_path = os.path.join(weight_file + "(test)")196        print(f"result {cnn_result_path}")197        if not os.path.exists(keras_util.get_prediction_path(cnn_result_path)):198            print(cnn_result_path)199            attr_get_model, attr_model_config = keras_util.dynamic_model_import(weight_file)200            model = attr_get_model(output_dim=len(attr_model_config.label_position), weights=None)201            model.load_weights(weight_file)202            attr_model_config.val_files = []203            for data_type in attr_model_config.data_type:204                if data_type == path.DATA_TYPE_ORIGINAL:205                    attr_model_config.val_files.append(original_test_file)206207            attr_model_config.tta_flip = True208            y_pred = keras_util.predict_tta(model, attr_model_config, verbose=1)209210            for i in range(13):211                y_pred[:, i] = y_pred[:, i] > thresholds[i][weight_file]212213            y_pred = y_pred.astype(np.int8)214215            keras_util.save_prediction_file(y_pred, cnn_result_path)216        else:217            y_pred = np.load(keras_util.get_prediction_path(cnn_result_path))218219        build_epoch_test(y_true, y_pred, weight_file)220221222def calc_xgb_f2_score():223    y_true = get_test_labels()224    y_pred = get_xgb_result()225226    with open(os.path.join(path.RESULT_PATH, "test.txt"), "a+") as f:227        one_label_greedy_f2_all = []228229        for i in range(13):230            one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))231232        f.write("\n\n")233        f.write("Greedy F2-Score is: %f\n" % np.average(one_label_greedy_f2_all))234        for i in range(13):235            f.write("[label %d] greedy-f2=%4f\n" % (i, one_label_greedy_f2_all[i]))236237238def get_existed_cnn_f2_score(val, mode_path):239    y_true = get_test_labels()240241    original_test_file = []242    cnt = 0243    with open(path.TEST_DATA_TXT, 'r') as f:244        for i in f.readlines():245            image_name = i.split(",")[0] + ".jpg"246            original_test_file.append(os.path.join(path.ORIGINAL_TEST_IMAGES_PATH, image_name))247248    weight_files = []249    _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)250251    for root, dirs, files in os.walk(mode_path):252        for file in files:253            if "predict.np" not in file or f"val{val}" not in root:254                continue255            weight_files.append(os.path.join(root, file)[:-12])256257    result = [{} for i in range(13)]258    average = {}259260    with open(os.path.join(path.RESULT_PATH, f"test(training)(all)(val{val}).txt"), "w+") as f:261        for weight_file in weight_files:262            print(f"weight file {weight_file}")263            unique_path = re.match(r".*competition[\\/]*(.*)", weight_file).group(1)264            identifier = "-".join(unique_path.split("\\"))265            print(f"id {identifier}")266            cnn_result_path = os.path.join(path.CNN_RESULT_PATH, identifier)267            print(f"result {cnn_result_path}")268            if os.path.exists(keras_util.get_prediction_path(cnn_result_path)):269                y_pred = np.load(keras_util.get_prediction_path(cnn_result_path))270            else:271                print("model not existed")272                continue273274            if thresholds[0].get(weight_file) is None:275                print("threshold not existed")276                continue277278            for i in range(13):279                y_pred[:, i] = y_pred[:, i] > thresholds[i][weight_file]280281            y_pred = y_pred.astype(np.int8)282283            one_label_greedy_f2_all = []284285            for i in range(13):286                # f2, _  = metrics.greedy_f2_score(y_true[:, i], y_pred[:, i], 1)287                f2 = fbeta_score(y_true[:, i], y_pred[:, i], beta=2)288                one_label_greedy_f2_all.append(f2)289                result[i][weight_file] = f2290            average[weight_file] = np.mean(one_label_greedy_f2_all)291292            f.write("\n\n")293            f.write("Weight: %s\n" % cnn_result_path)294            f.write("Greedy F2-Score is: %f\n" % np.mean(one_label_greedy_f2_all))295            for i in range(13):296                f.write("[label %d] greedy-f2=%4f\n" % (i, one_label_greedy_f2_all[i]))297298            print(f"need predict {cnt} model")299300    with open(os.path.join(path.RESULT_PATH, f"test(training)(label)(val{val}).txt"), "w+") as f:301        average = sorted(average.items(), key=lambda x: x[1], reverse=True)302        f.write(f"\n\n==================== all =======================\n\n")303        for item in average:304            f.write("%4f: %s \n" % (item[1], item[0]))305306        for i in range(13):307            dic = result[i]308            dic = sorted(dic.items(), key=lambda x: x[1], reverse=True)309            f.write(f"\n\n====================label {i} =======================\n\n")310            for item in dic:311                f.write("%4f: %s \n" % (item[1], item[0]))312313314def build_epoch_cv(val):315    all_label, one_label, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)316    all_f2 = get_epoch_cv()317318    for all in all_label:319        f2 = {}320        f2["avg"] = all[1]321        all_f2[get_epoch_identifier(all[0])] = f2322323    for label in range(13):324        for one in one_label[label]:325            identifier = get_epoch_identifier(one[0])326            all_f2[identifier][f"{label}"] = one[1]327328    save_epoch_cv(all_f2)329330331def build_model_cv(val):332    all_label, one_label, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)333    all_f2 = get_model_cv()334335    for all in all_label:336        f2 = {}337        f2["avg"] = all[1]338        for i in range(13):339            f2[f"{i}"] = 0340        all_f2[get_model_identifier(all[0])] = f2341342    for label in range(13):343        for one in one_label[label]:344            identifier = get_model_identifier(one[0])345            if one[1] > all_f2[identifier][f"{label}"]:346                all_f2[identifier][f"{label}"] = one[1]347                all_f2[identifier][f"model{label}"] = get_epoch_identifier(one[0])348349    for key in all_f2.keys():350        avg = 0351        for i in range(13):352            avg += all_f2[key][f"{i}"]353        avg /= 13354        all_f2[key]["avg"] = avg355356    save_model_cv(all_f2)357358359def build_model_test():360    epoch_test = get_epoch_test()361    model_cv = get_model_cv()362    model_test = {}363    for id in epoch_test.keys():364        model_test[id.split(".")[0]] = {"avg": 0}365        for i in range(13):366            model_test[id.split(".")[0]][f"{i}"] = 0367368    for id in model_cv:369        value = model_cv[id]370        avg = 0371        for i in range(13):372            model = value[f"model{i}"]373            if epoch_test.get(model):374                test_model = epoch_test[model]375                test_f2 = test_model[f"{i}"]376                model_test[id.split(".")[0]][f"{i}"] = test_f2377                model_test[id.split(".")[0]][f"model{i}"] = model378                avg += test_f2379                model_test[id.split(".")[0]]["avg"] = avg / 13380381    model_test_sorted = sorted(model_test.items(), key=lambda x: x[1]["avg"], reverse=True)382    model_test = {}383    for item in model_test_sorted:384        model_test[item[0]] = item[1]385386    save_model_test(model_test)387388    print("ok")389390391def build_threshold_cv():392    threshold_cv = {}393394    for val in range(1, 6):395        _, _, thresholds = model_statistics.model_f2_statistics(path.MODEL_PATH, val)396        print("ok")397398        for label in range(13):399            for weight_path in thresholds[label].keys():400                if not threshold_cv.get(get_epoch_identifier(weight_path)):401                    threshold_cv[get_epoch_identifier(weight_path)] = {}402                threshold_cv[get_epoch_identifier(weight_path)][f"{label}"] = thresholds[label][weight_path]403        print("ok")404405    save_threshold_cv(threshold_cv)406407def build_global_cv():408    epoch_cv = get_epoch_cv()409    global_cv = {"avg": 0}410    for i in range(13):411        global_cv[f"{i}"] = 0412413    for id in epoch_cv.keys():414        model = epoch_cv[id]415        for label in range(13):416            if model[f"{label}"] > global_cv[f"{label}"]:417                global_cv[f"{label}"] = model[f"{label}"]418                global_cv[f"model{label}"] = id419420    avg = 0421    for i in range(13):422        avg += global_cv[f"{i}"]423    global_cv["avg"] = avg / 13424    save_global_cv(global_cv)425426427def build_global_test():428    global_cv = get_global_cv()429    epoch_test = get_epoch_test()430    global_test = {"avg": 0}431    for i in range(13):432        global_test[f"{i}"] = 0433434    for label in range(13):435        cv_model = global_cv[f"model{label}"]436        test_model = epoch_test[cv_model]437        f2 = test_model[f"{label}"]438        global_test[f"{label}"] = f2439440    avg = 0441    for i in range(13):442        avg += global_test[f"{i}"]443        global_test["avg"] = avg / 13444445    save_global_test(global_test)446447448def cnn_result_name_to_epoch_name(cnn: str):449    cnn = cnn[:-12]450    cnn = cnn.replace("-", "\\")451    return "\\" + cnn452453454def build_ensemble_epoch_cv():455    result_paths = []456    result_files = {}457458    threshold_cv = get_threshold_cv()459460    for root, dirs, files in os.walk(path.CNN_RESULT_PATH):461        for file in files:462            if "val2" not in file:463                continue464            if file.split(".")[-1] != "npy":465                continue466            result_paths.append(os.path.join(root, file))467            result_files[os.path.join(root, file)] = file468469    for result_path in result_paths:470        file_name = result_files[result_path]471        epoch_name = cnn_result_name_to_epoch_name(file_name)472        y_true = get_test_labels()473        y_pred = np.load(result_path)474475        threshold = threshold_cv[epoch_name]476477        for i in range(13):478            y_pred[:, i] = y_pred[:, i] > threshold[f"{i}"]479480        y_pred = y_pred.astype(np.int8)481482        build_epoch_test(y_true, y_pred, None, identifier=epoch_name)483484485def build_threshold_test():486    result_paths = []487    result_files = {}488489    threshold_test = {}490491    for root, dirs, files in os.walk(path.CNN_RESULT_PATH):492        for file in files:493            if "val2" not in file:494                continue495            if file.split(".")[-1] != "npy":496                continue497            result_paths.append(os.path.join(root, file))498            result_files[os.path.join(root, file)] = file499500    for result_path in result_paths:501        file_name = result_files[result_path]502        epoch_name = cnn_result_name_to_epoch_name(file_name)503504        if threshold_test.get(epoch_name):505            continue506        print(f"epoch name {epoch_name}")507        y_true = get_test_labels()508        y_pred = np.load(result_path)509        threshold_test[epoch_name] = {}510        for i in range(13):511            score, threshold = metrics.greedy_f2_score(y_true[:, i], y_pred[:, i], 1, step=100)512            threshold_test[epoch_name][f"{i}"] = threshold[0]513514        save_threshold_test(threshold_test)515516517def build_epoch_test_standard():518    result_paths = []519    result_files = {}520521    threshold_test = get_threshold_test()522    epoch_test_standard = {}523524    for root, dirs, files in os.walk(path.CNN_RESULT_PATH):525        for file in files:526            if "val2" not in file:527                continue528            if file.split(".")[-1] != "npy":529                continue530            result_paths.append(os.path.join(root, file))531            result_files[os.path.join(root, file)] = file532533    for result_path in result_paths:534        file_name = result_files[result_path]535        epoch_name = cnn_result_name_to_epoch_name(file_name)536        threshold = threshold_test[epoch_name]537        y_true = get_test_labels()538        y_pred = np.load(result_path)539540        for i in range(13):541            y_pred[:, i] = y_pred[:, i] > threshold[f"{i}"]542543        y_pred = y_pred.astype(np.int8)544545        one_label_greedy_f2_all = []546        for i in range(13):547            one_label_greedy_f2_all.append(fbeta_score(y_true[:, i], y_pred[:, i], beta=2))548549        f2 = {"avg": np.mean(one_label_greedy_f2_all)}550        for i in range(13):551            f2[f"{i}"] = one_label_greedy_f2_all[i]
...

model_analytics.py

Source:model_analytics.py

...14    def get_model(self):15        return self.model16    def get_predictions(self):17        return self.predictions18    def get_test_labels(self):19        return self.test_labels20    def get_feature_list(self):21        return self.feature_list22    def get_feat_imp(self):23        return self.rf_feat_imp24    def save_model(self,file_name):25        try:26            pickle.dump(self.get_model(),open(file_name + ".pickle",'wb'))27            print ("Model successfully saved.")28        except Exception as e:29            print ("Model not saved", e)30    def model_absolute_errors(self):31        # Calculate the absolute errors32        return abs(self.get_predictions() - self.get_test_labels())33    def model_mean_absolute_error(self):34        # Return the mean absolute error (mae)35        return round(np.mean(self.model_absolute_errors()), 4)36    def model_mean_absolute_percentage_error(self):37        # Calculate mean absolute percentage error (MAPE)38        return 100 * (self.model_absolute_errors() / self.get_test_labels())39    def model_accuracy(self):40        # Calculate and display accuracy41        return 100 - np.mean(self.model_absolute_errors())42    def model_importances(self):43        return list(self.get_feat_imp())44    def model_feature_importance(self):45        # List of tuples with variable and importance46        imp_tmp = self.model_importances()47        feat_list_tmp = self.get_feature_list()48        feature_importance = [(feature, round(imp_tmp, 2)) for feature, imp_tmp in zip(feat_list_tmp, imp_tmp)]49        # Sort the feature importances by most important first50        feature_importance = sorted(feature_importance, key=lambda x: x[1], reverse=True)51        # Print out the feature and importances52        [print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importance]...

utils.py

Source:utils.py

1import os2import cv23import glob4import random5import numpy as np6from itertools import chain7def read_image(img_path, is_resize=True, resize_dim=64):8	"""9	This function will read each individual image. The image will be converted to gray scale. Finally if specified,10	the image will be resized and returned11	:param img_path: the path of the image to be read12	:param is_resize: Weather to resize the image13	:param resize_dim: The height and width of the resized image14	:return: resized_img15	"""16	img = cv2.imread(img_path)17	img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)18	if is_resize:19		img = cv2.resize(img, (resize_dim, resize_dim))20	return img21def read_directory(img_paths, str_labels, is_test=False):22	"""23	This function will return the numpy images data and the corresponding labels for the image paths provided.24	:param img_paths: the image paths to be read to images25	:param str_labels:26	:param is_test: defines if we are running code in test or train mode27	:return: classes, resized_img_data28	"""29	img_data = np.array([read_image(x) for x in img_paths], dtype=np.uint8)30	if is_test:31		return img_data, None32	labels = np.array([str_labels.index(x.split('/')[-1].split('\\')[0]) for x in img_paths], dtype=np.int)33	return img_data, labels34def get_clf_data(path, train_per=0.8, is_split=True, get_test_labels=True):35	"""36	This function will read the entire data from the specified directory. The function will return the image data and37	their corresponding classes.38	:param path: The root directory path of the images39	:param train_per: if split, the percentage of images to be put into train40	:param is_split: defines if data is to be split in train and test41	:param get_test_labels: whether test labels are to be returned.42	:return:43	"""44	str_labels = os.listdir(path)45	img_paths = [*chain(*[glob.glob(path + '/{}/*.png'.format(x)) for x in str_labels])]46	random.shuffle(img_paths)47	if is_split:48		n_imgs = len(img_paths)49		n_train = int(train_per * n_imgs)50		train_path, test_path = img_paths[:n_train], img_paths[n_train:]51		train_data, train_labels = read_directory(train_path, str_labels)52		test_data, test_labels = read_directory(test_path, str_labels, is_test=(not get_test_labels))53		np.save('data/str_train_labels.npy', str_labels)54		return train_data, test_data, train_labels, test_labels55	else:56		return read_directory(img_paths, str_labels, is_test=(not get_test_labels))57if __name__ == '__main__':58	data_path = '../../data/fish_dataset/data'59	tr_data, te_data, tr_labels, te_labels = get_clf_data(data_path)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.