How to use annotate_file method in autotest

Best Python code snippet using autotest_python

EvalModel.py

Source:EvalModel.py

1#!/usr/bin/env python2# -*- coding: utf-8 -*-3import sys4# from analysis import analysis5import csv6import scipy7from torch.utils.data import DataLoader8import math9from sentence_transformers import SentenceTransformer10from sentence_transformers import models11import logging12from datetime import datetime13import sys14import os15import argparse16import logging17import hashlib18sys.path.append("..")19from analysis import analysis20import argparse21logging.basicConfig(level=logging.DEBUG, format="[%(levelname).1s %(asctime)s] %(message)s", datefmt="%Y-%m-%d_%H:%M:%S")22def load_annotate_data(annotate_file, delimiter='\t'):23    annotate_dataset = []24    with open(annotate_file,"r") as fp:25        for line in fp:26            items = line.strip().split(delimiter)27            annotate_dataset.append(items)28    return annotate_dataset29def load_title_content_data(annotate_file, delimiter='\t'):30    annotate_dataset = []31    with open(annotate_file,"r", encoding="utf-8") as fp:32        for line in fp:33            line = line.encode("utf-8").decode("unicode_escape")34            items = line.rstrip('\n').split(delimiter)35            if len(items) != 5:36                print(line)37            kw, title, content, label, docid = items38            annotate_dataset.append([kw, title + ' ' + content, label])39    return annotate_dataset40def eval_model(annotate_file, model_name, eval_res_file):41    # annotate_dataset = load_annotate_data(annotate_file)42    annotate_dataset = load_title_content_data(annotate_file)43    results = []44    idindex = {}45    corpus = []46    count = 047    for query, sen, label in annotate_dataset:48        id_gen = hashlib.md5()49        id_gen.update(query.encode('utf-8'))50        query_id = id_gen.hexdigest()51        if query_id not in idindex:52            corpus.append(query)53            idindex[query_id] = count54            count += 155        id_gen = hashlib.md5()56        id_gen.update(sen.encode('utf-8'))57        sen_id = id_gen.hexdigest() 58        if sen_id not in idindex:59            corpus.append(sen)60            idindex[sen_id] = count61            count += 162    63    model = SentenceTransformer(model_name)64    #word_embedding_model = models.Transformer(model_name)65    #pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension(),66    #                           pooling_mode_mean_tokens=True,67    #                           pooling_mode_cls_token=False,68    #                           pooling_mode_max_tokens=False)69    #model = SentenceTransformer(modules=[word_embedding_model, pooling_model])70    corpus_embeddings = model.encode(corpus)71    for query, sen, label in annotate_dataset:72        label = int(label)73        id_gen = hashlib.md5()74        id_gen.update(query.encode('utf-8'))75        query_id = id_gen.hexdigest()76        id_gen = hashlib.md5()77        id_gen.update(sen.encode('utf-8'))78        sen_id = id_gen.hexdigest() 79        logging.debug('query:%s'%(query))80        logging.debug('idindex[query_id]:%d'%(idindex[query_id]))81        logging.debug('sen:%s'%(sen))82        logging.debug('idindex[sen_id]:%d'%(idindex[sen_id]))83        query_vec = corpus_embeddings[idindex[query_id]]84        sen_vec = corpus_embeddings[idindex[sen_id]]85        sim_score = scipy.spatial.distance.cdist([query_vec],[sen_vec], "cosine")[0] 86        results.append((label,query_id, sim_score, sen_id))87    fp = open(eval_res_file,"w", encoding="utf-8")88    writer = csv.writer(fp)89    ndcg = analysis.cal_NDCG(results,10)90    writer.writerow([model_path, ndcg])91    fp.close()92def model_predict(input_file, output_file):93    pass94if __name__ == "__main__":95    os.environ["CUDA_VISIBLE_DEVICES"] = "1"96    parser = argparse.ArgumentParser(description="Text Similarity")97    parser.add_argument('--annotate_file', action='store', type=str, required=True, help="annotate_file")98    parser.add_argument('--model_path', required=True, type=str, help="model_path")99    parser.add_argument('--eval_res_file', default="eval_res_file", type=str, help="eval_res_file")100    args = parser.parse_args()101    annotate_file = args.annotate_file102    model_path = args.model_path103    eval_res_file = args.eval_res_file104    eval_model(annotate_file, model_path, eval_res_file)...

annotate.py

Source:annotate.py

...27        """Run the report.28        See `coverage.report()` for arguments.29        """30        self.report_files(self.annotate_file, morfs, directory)31    def annotate_file(self, cu, analysis):32        """Annotate a single file.33        `cu` is the CodeUnit for the file to annotate.34        """35        if not cu.relative:36            return37        filename = cu.filename38        source = cu.source_file()39        if self.directory:40            dest_file = os.path.join(self.directory, cu.flat_rootname())41            dest_file += ".py,cover"42        else:43            dest_file = filename + ",cover"44        dest = open(dest_file, 'w')45        statements = sorted(analysis.statements)...

pretrain_custom.py

Source:pretrain_custom.py

1import torch2from PIL import Image3import pandas as pd4class CustomData(torch.utils.data.Dataset):5	def __init__(self , annotate_file , transform = None):6		self.annotate_file = pd.read_csv(annotate_file)7		self.transform = transform8	def __len__(self):9		return len(self.annotate_file)10	def __getitem__(self , index):11		label = int(self.annotate_file.iloc[index , 1])12		img = Image.open(self.annotate_file.iloc[index , 0])13		if self.transform:14			img = self.transform(img)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.