Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use cat_eval method in fMBT

Best Python code snippet using fMBT_python

cat_train.py

Source:cat_train.py

1# -*- coding: utf-8 --*--2# @Author: Zessay3# @time: 2019.05.09 9:494# @File: cat_train.py5# @Software: PyCharm67import os8import catboost9from catboost import Pool10from utils import sort_val, find_threshold, save_pkl, return_entity, save_as_order11from config import DefaultConfig121314class CATTrain:15    def __init__(self, train, test, params):16        self.train_ = train17        self.test_ = test18        self.params = params19        self.threshold = None20        self.model = None21        self.fe = None22        self.pred = None23        self.opt = DefaultConfig()2425    # è®ç»é¨åè®ç»éæ°æ®ï¼å¨éªè¯éæç´¢åå²éå¼26    def train_for_threshold(self, features, target='label', num=35000):27        train_df = self.train_[self.train_.ID < num]28        val_df = self.train_[self.train_.ID >= num]2930        X_train, y_train = train_df[features].values, train_df[target].values.astype('uint8')31        X_eval, y_eval = val_df[features].values, val_df[target].values.astype('uint8')3233        cat_train = Pool(X_train, y_train)34        cat_eval = Pool(X_eval, y_eval)3536        cat_model = catboost.train(cat_train, self.params, iterations=10000,37                              eval_set=cat_eval,38                              early_stopping_rounds=200,39                              verbose=500)40        y_pred = cat_model.predict(cat_eval, prediction_type='Probability')[:,1]41        ## è·åéªè¯éççå®å®ä½ï¼ä»¥åæé¡ºåºæåºé¢æµçæ¦çåå¯¹åºçåè¯42        gt_ent, pred_words, pred_proba = sort_val(val_df, y_pred)43        ## è·åæç´¢å¾å°çéå¼ç»æ44        self.threshold, _ = find_threshold(gt_ent, pred_words, pred_proba)4546        return self.threshold4748    # è®ç»å¨é¨æ°æ®49    def train_and_predict(self, features, target='label', save=True):50        self.fe = features51        X_train, y_train = self.train_[features].values, self.train_[target].values.astype('uint8')52        X_test = self.test_[self.fe].values5354        cat_all = Pool(X_train, y_train)5556        model = catboost.train(57            cat_all,58            self.params,59            iterations=10000,60            early_stopping_rounds=200,61            verbose=100062        )6364        self.model = model6566        if save:67            save_pkl(model, os.path.join(self.opt['model_train'], 'cat.pkl'))6869        self.pred = model.predict(X_test, prediction_type='Probability')[:, 1]7071        return self.pred7273    # ä¿åæµè¯éå¾å°çç»æ74    def save_result(self):75        pred_id, pred_words, pred_proba = sort_val(self.test_, self.pred, predict=True)76        entities = return_entity(pred_words, pred_proba, self.threshold)7778        save_as_order(pred_id, entities, self.opt, 'cat_result.txt')7980818283
...

eval.py

Source:eval.py

1"""2run evaluation on test dataset3"""4import argparse5import json6from pycocotools.coco import COCO7from pycocotools.cocoeval import COCOeval8import pandas as pd9from .data.coco import category_count10def coco_eval(cocoGt, cocoDt, catIds):11    E = COCOeval(cocoGt, cocoDt, iouType='bbox')  # initialize CocoEval object12    if catIds:13        E.params.catIds = catIds14        print([cocoGt.cats[c] for c in E.params.catIds])15    E.evaluate()16    E.accumulate()17    E.summarize()18    return E.stats19if __name__ == '__main__':20    parser = argparse.ArgumentParser(description=__doc__,21                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter,22                                     fromfile_prefix_chars='@')23    parser.add_argument("--gt", type=str, help='ground truth annotation')24    parser.add_argument('--dt', type=str, default='submission.json', help='detection json')25    parser.add_argument('--dt-images', action='store_true', help='use images of detection only')26    parser.add_argument('--catId', type=lambda s: [int(i) for i in s.split(',')], default=[], help='select catIds')27    parser.add_argument('--each-category', action='store_true', help='evaluate each category')28    args = parser.parse_args()29    cocoGt = COCO(args.gt)30    if args.dt_images:31        dt = json.load(open(args.dt))32        dt_images = set([d['image_id'] for d in dt])33        print('#images:', len(dt_images))34        cocoGt.imgs = {k: v for k, v in cocoGt.imgs.items() if k in dt_images}35    cocoDt = cocoGt.loadRes(args.dt)36    catIds = cocoGt.getCatIds(catIds=args.catId)37    if args.each_category:38        cat_count = category_count(cocoGt)39        print('cat_count', cat_count)40        cat_eval = {c: coco_eval(cocoGt, cocoDt, [c]) for c in catIds}41        cat_eval = {c: v for c, v in cat_eval.items() if (v >= 0).any()}42        columns = ['mAP', 'AP50', 'AP75', 'mAPs', 'mAPm', 'mAPl', 'AR1', 'AR10', 'AR', 'ARs', 'ARm', 'ARl', 'T', 'name']43        values = [v.tolist() + [cat_count[k], cocoGt.cats[k]['name']] for k, v in cat_eval.items()]44        cat_eval = pd.DataFrame(values, index=cat_eval.keys(), columns=columns)45        print("=============================================== summary ===============================================")46        pd.set_option('display.max_columns', None)47        pd.set_option('display.max_rows', None)48        pd.set_option('display.precision', 3)49        pd.set_option('display.width', 120)50        print(cat_eval.sort_values(by=['mAP']))...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.