Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use test_results_path method in avocado

Best Python code snippet using avocado_python

eval.py

Source:eval.py

1#2# Based on Sebastian HofstÃ¤tter code published in https://github.com/sebastian-hofstaetter/matchmaker3#45import os6import copy7import time8import glob9from typing import Dict, Tuple, List10os.environ["TOKENIZERS_PARALLELISM"] = "false"1112import numpy as np13import scipy.stats as stats1415from evaluation.utils import *16import time171819def read_bm25(test_results_path):20    with open(test_results_path, 'r') as file:21       lines = file.readlines()22       test_results = {}23       for line in lines:24           splitted_line = line.split(' ')25           query_id = splitted_line[0]26           doc_id = splitted_line[2]27           score = float(splitted_line[4])28           if test_results.get(query_id):29               doc_list = test_results.get(query_id)30               doc_list.append((doc_id, score))31               test_results.update({query_id:doc_list})32           else:33               test_results.update({query_id:[(doc_id, score)]})34    return test_results3536def read_run(test_results_path):37    with open(test_results_path, 'r') as file:38        lines = file.readlines()39        test_results = {}40        for line in lines:41            splitted_line = line.split('\t')42            query_id = splitted_line[0]43            doc_id = splitted_line[1]44            score = float(splitted_line[3])45            if test_results.get(query_id):46                doc_list = test_results.get(query_id)47                doc_list.append((doc_id, score))48                test_results.update({query_id:doc_list})49            else:50                test_results.update({query_id:[(doc_id, score)]})51    return test_results525354def eval_run(test_results_path, qrels_path, binarization_point, qrels_with_irrel=False, with_j=False):55    if 'bm25' in test_results_path:56        test_results = read_bm25(test_results_path)57    else:58        test_results = read_run(test_results_path)5960    metrics = None61    ranked_results = unrolled_to_ranked_result(test_results)6263    if with_j:64        # for j coverage we need all the irrelevant and relevant judgements65        qrels = load_qrels_with_irrel(qrels_path)66        ranked_results_j = {}67        for query_id, value in qrels.items():68            if ranked_results.get(query_id):69                doc_ids_ranked_list = ranked_results.get(query_id)70                ranked_list_per_query_id = []71                for doc_id in doc_ids_ranked_list:72                    if doc_id in list(value.keys()):73                        ranked_list_per_query_id.append(doc_id)74                ranked_results_j.update({query_id: ranked_list_per_query_id})7576        ranked_results = ranked_results_j77        # but for the evaluation of nDCG only the positive judgements should be in the qrels set78        qrels = load_qrels(qrels_path)79    elif qrels_with_irrel:80        qrels = load_qrels_with_irrel(qrels_path)81    else:82        qrels = load_qrels(qrels_path)8384    metrics = calculate_metrics_plain(ranked_results,qrels,binarization_point)8586    if 'dctr' in qrels_path:87        qrels = 'dctr'88    elif 'raw' in qrels_path:89        qrels = 'raw'90    else:91        qrels = 'annotation'9293    metric_file_path = os.path.join('/'.join(test_results_path.split('/')[:-1]), 'test_top200_rerank_head_{}_joption'.format(qrels)+"-metrics.csv")94    save_fullmetrics_oneN(metric_file_path, metrics, -1, -1)95    return metrics969798def get_metrics_for_multiple_qrels_runs(qrels, runs, path, binarization_point):99    metrics = {}100    for qrel in qrels:101        if 'dctr' in qrel:102            qrel_name = 'dctr'103        elif 'raw' in qrel:104            qrel_name = 'raw'105        else:106            qrel_name = 'annotation'107        metrics.update({qrel_name: {}})108        for run in runs:109            test_results_path = os.path.join(path, run)110            run_name = run.split('/')[0]111112            run_metrics = eval_run(test_results_path, qrel, binarization_point)113            metrics.get(qrel_name).update({run_name: run_metrics})114    return metrics115116117def compute_kendalls_tau_between_collections(pairs, measures, metrics, path):118    with open(os.path.join(path, 'kendalltau.txt'), 'w') as f:119        for pair in pairs:120            for measure in measures:121                runs = metrics.get(pair[0])122                measure_numbers = []123                for value in runs.values():124                    measure_numbers.append(value.get(measure))125                runs_per_measure = dict(zip(runs.keys(), measure_numbers))126                runs_per_measure_sorted_1 = dict(127                    sorted(runs_per_measure.items(), key=lambda item: item[1], reverse=True))128129                runs = metrics.get(pair[1])130                measure_numbers = []131                for value in runs.values():132                    measure_numbers.append(value.get(measure))133                runs_per_measure = dict(zip(runs.keys(), measure_numbers))134                runs_per_measure_sorted_2 = dict(135                    sorted(runs_per_measure.items(), key=lambda item: item[1], reverse=True))136137                tau, p_value = stats.kendalltau(list(runs_per_measure_sorted_1.keys()),138                                                list(runs_per_measure_sorted_2.keys()))139                print('pair {} measure {} and kendall tau {} pvalue {}'.format(pair, measure, tau, p_value))140141                f.write('pair {} measure {} and kendall tau {} pvalue {}\n'.format(pair, measure, tau, p_value))142        f.close()143144145def compute_kendalls_tau_between_metrics(pairs, tests, metrics, path):146    with open(os.path.join(path, 'kendalltau_withinstability_ndcg3.txt'), 'w') as f:147        for test in tests:148            for pair in pairs:149                runs = metrics.get(test)150                measure_numbers = []151                for value in runs.values():152                    measure_numbers.append(value.get(pair[0]))153                runs_per_measure = dict(zip(runs.keys(), measure_numbers))154                runs_per_measure_sorted_1 = dict(155                    sorted(runs_per_measure.items(), key=lambda item: item[1], reverse=True))156157                runs = metrics.get(test)158                measure_numbers = []159                for value in runs.values():160                    measure_numbers.append(value.get(pair[1]))161                runs_per_measure = dict(zip(runs.keys(), measure_numbers))162                runs_per_measure_sorted_2 = dict(163                    sorted(runs_per_measure.items(), key=lambda item: item[1], reverse=True))164165                tau, p_value = stats.kendalltau(list(runs_per_measure_sorted_1.keys()),166                                                list(runs_per_measure_sorted_2.keys()))167                print('pair {} test {} and kendall tau {} pvalue {}'.format(pair, test, tau, p_value))168169                f.write('pair {} test {} and kendall tau {} pvalue {}\n'.format(pair, test, tau, p_value))170        f.close()171172if __name__ == "__main__":173    test_results_path = 'path_to/ensemble-output.txt'174    qrels_path_anno = './data/qrels_2class.txt'175    qrels_path_dctr = 'path_to/qrels.dctr.head.test.txt'176    qrels_path_raw = 'path_to/qrels.raw.head.test.txt'177    binarization_point = 1178179    runs = ['bm25/bm25_top1k_head.test.txt',180           'scibert_dot/top1k_head_dctr-output.txt',181           'pubmedbert_dot/top1k_head_dctr-output.txt',182           'colbert_scibert/test_top200_rerank_head_dctr-output.txt',183           'colbert_pubmedbert/test_top200_rerank_head_dctr-output.txt',184           'bert_cat/test_top200_rerank_head_dctr-output.txt',185           '3bert_fix_ensemble_avg/top200_rerank_head_dctr-ensemble-output.txt']186187    path = 'output_path/'188    qrels = [qrels_path_anno, qrels_path_dctr, qrels_path_raw]189190    # evaluate all runs again with the irrelevant ones to find coverage!191    for qrel in qrels:192        for run in runs:193            test_results_path = os.path.join(path, run)194            eval_run(test_results_path, qrel, binarization_point, qrels_with_irrel=True, with_j=True)195196    # evaluate annotated runs with joption197    for run in runs:198        test_results_path = os.path.join(path, run)199        eval_run(test_results_path, qrels_path_anno, binarization_point, qrels_with_irrel=True, with_j=True)200201202    # evaluate multiple runs for kendalls tau203    metrics = get_metrics_for_multiple_qrels_runs(qrels, runs, path, binarization_point)204    measures = ['Recall@100', 'MRR@10', 'nDCG@5', 'nDCG@10', 'nDCG@3']205    pairs = [('annotation', 'dctr'), ('annotation', 'raw'), ('dctr', 'raw')]206207    compute_kendalls_tau_between_collections(pairs, measures, metrics, path)208209    # stability of test collection: kendalls tau between metrics210    measures = ['Recall@100', 'MRR@10', 'nDCG@5', 'nDCG@10', 'nDCG@3']211    pairs = [('nDCG@3', 'Recall@100'), ('nDCG@3', 'MRR@10'), ('nDCG@3', 'nDCG@5')]212    tests = ['annotation', 'raw', 'dctr']213    compute_kendalls_tau_between_metrics(pairs, tests, metrics, path)214
...

test_mtom.py

Source:test_mtom.py

1"""2CEASIOMpy: Conceptual Aircraft Design Software3Developed by CFS ENGINEERING, 1015 Lausanne, Switzerland4Test functions for 'ceasiompy/WeightConventional/func/mtom.py'5Python version: >=3.76| Author : Aidan Jungo7| Creation: 2022-05-308"""9# =================================================================================================10#   IMPORTS11# =================================================================================================12from pathlib import Path13import shutil14from pytest import approx15from ceasiompy.WeightConventional.func.mtom import estimate_mtom16from ceasiompy.utils.commonnames import MTOM_FIGURE_NAME17MODULE_DIR = Path(__file__).parent18TEST_RESULTS_PATH = Path(MODULE_DIR, "ToolOutput")19# =================================================================================================20#   CLASSES21# =================================================================================================22# =================================================================================================23#   FUNCTIONS24# =================================================================================================25def test_estimate_mtom():26    """Test function 'estimate_mtom'"""27    if TEST_RESULTS_PATH.exists():28        shutil.rmtree(TEST_RESULTS_PATH)29    TEST_RESULTS_PATH.mkdir()30    # estimate_mtom(fuselage_length, fuselage_width, wing_area, wing_span)31    # These tests are made from random value just to check if the function return the same kind of32    # result after it will be refactored.33    assert estimate_mtom(50, 5, 750, 60, TEST_RESULTS_PATH) == approx(223398, rel=1e-2)34    assert estimate_mtom(55, 5, 750, 60, TEST_RESULTS_PATH) == approx(231389, rel=1e-2)35    assert estimate_mtom(50, 5, 550, 60, TEST_RESULTS_PATH) == approx(176957, rel=1e-2)36    assert estimate_mtom(50, 5, 550, 50, TEST_RESULTS_PATH) == approx(173153, rel=1e-2)37    assert estimate_mtom(20, 2.5, 250, 28, TEST_RESULTS_PATH) == approx(23255, rel=1e-2)38    assert estimate_mtom(12, 2, 65, 15, TEST_RESULTS_PATH) == approx(18900, rel=1e-2)39    assert estimate_mtom(12, 2, 62, 15, TEST_RESULTS_PATH) == approx(17707, rel=1e-2)40    # Seems wrong, maybe out of range41    # assert estimate_mtom(10, 1.8, 50, 11, TEST_RESULTS_PATH) == approx(17707, rel=1e-2)42    # --> Obtained: 33795.11843    assert Path(TEST_RESULTS_PATH, MTOM_FIGURE_NAME).exists()44# =================================================================================================45#    MAIN46# =================================================================================================47if __name__ == "__main__":48    print("Running Test WeightConventional")49    print("To run test use the following command:")...

test_oem.py

Source:test_oem.py

1"""2CEASIOMpy: Conceptual Aircraft Design Software3Developed by CFS ENGINEERING, 1015 Lausanne, Switzerland4Test functions for 'ceasiompy/WeightConventional/func/mtom.py'5Python version: >=3.76| Author : Aidan Jungo7| Creation: 2022-05-308"""9# =================================================================================================10#   IMPORTS11# =================================================================================================12import shutil13from pathlib import Path14from ceasiompy.WeightConventional.func.oem import estimate_oem15from pytest import approx16MODULE_DIR = Path(__file__).parent17TEST_RESULTS_PATH = Path(MODULE_DIR, "ToolOutput")18# =================================================================================================19#   CLASSES20# =================================================================================================21# =================================================================================================22#   FUNCTIONS23# =================================================================================================24def test_estimate_oem():25    """Test function 'estimate_oem'"""26    if TEST_RESULTS_PATH.exists():27        shutil.rmtree(TEST_RESULTS_PATH)28    TEST_RESULTS_PATH.mkdir()29    # estimate_oem(mtom, fuse_length, wing_span, turboprop)30    assert estimate_oem(350_000, 58, 61, False) == approx(171_452, rel=1e-2)31    assert estimate_oem(250_000, 48, 51, False) == approx(125_807, rel=1e-2)32    assert estimate_oem(150_000, 38, 44, False) == approx(78_633, rel=1e-2)33    assert estimate_oem(150_000, 38, 44, True) == approx(76_690, rel=1e-2)34    assert estimate_oem(50_000, 24, 34, False) == approx(26_431, rel=1e-2)35    assert estimate_oem(50_000, 24, 34, True) == approx(28_897, rel=1e-2)36# =================================================================================================37#    MAIN38# =================================================================================================39if __name__ == "__main__":40    print("Running Test WeightConventional")41    print("To run test use the following command:")...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.