Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use _finalize_result method in Lemoncheesecake

Best Python code snippet using lemoncheesecake

eval.py

Source:eval.py

...40        final_batch = self._data.get_final_batch(batch_size, return_indices=self._batch_indices)41        if final_batch is not None:42            result = self._aggregate_batch(result, self._run_batch(model, final_batch))43        model.train()44        return self._finalize_result(result)45    @abc.abstractmethod46    def _run_batch(self, model, batch):47        """ Evaluates the model on a given batch of data """48    @abc.abstractmethod49    def _aggregate_batch(self, agg, batch_result):50        """ Aggregates batch result into total """51    @abc.abstractmethod52    def _initialize_result(self):53        """ Initializes the aggregate result """54    @abc.abstractmethod55    def _finalize_result(self, result):56        """ Returns the final value given the aggregate result """57    def get_name(self):58        return self._name59    @staticmethod60    def run_all(evaluations, model):61        results = dict()62        for evaluation in evaluations:63            if evaluation.get_trials() == 1:64                results[evaluation.get_name()] = evaluation.run(model)65            else:66                mean_name = evaluation.get_name()67                std_name = evaluation.get_name() + " std (n=" + str(evaluation.get_trials()) + ")"68                mean, std = evaluation.run(model)69                results[mean_name] = mean70                results[std_name] = std71        return results72class Loss(Evaluation):73    def __init__(self, name, data, data_parameters, loss_criterion, norm_dim=False, trials=1):74        super(Loss, self).__init__(name, data, data_parameters, trials=trials)75        self._loss_criterion = loss_criterion76        self._norm_dim = norm_dim77    def _run_batch(self, model, batch):78        loss = model.loss(batch, self._data_parameters, self._loss_criterion)79        if self._norm_dim:80            return (loss[0].data[0], loss[1].data[0])81        else:82            return loss.data[0]83    def _aggregate_batch(self, agg, batch_result):84        if self._norm_dim:85            return (agg[0] + batch_result[0], agg[1] + batch_result[1])86        else:87            return agg + batch_result88    def _initialize_result(self):89        if self._norm_dim:90            return (0.0, 0.0)91        else:92            return 0.093    def _finalize_result(self, result):94        if self._norm_dim:95            return result[0] / result[1]96        else:97            return result / self._data.get_size()98class DistributionAccuracy(Evaluation):99    def __init__(self, name, data, data_parameters, model_fn=None, target_indexed = False, check_unique = False, trials=1):100        super(DistributionAccuracy, self).__init__(name, data, data_parameters, trials=trials)101        self._model_fn = model_fn102        self._target_indexed = target_indexed103        self._check_unique = check_unique104    def _run_batch(self, model, batch):105        dist = None106        if self._model_fn is None:107            dist = model.forward_batch(batch, self._data_parameters)108        else:109            dist = self._model_fn(batch, model, self._data_parameters)110        target = batch[self._data_parameters[DataParameter.TARGET]].squeeze()111        model_ps = dist.p().data.cpu()112        max_ps, max_index = torch.max(model_ps, 1, keepdim=True)113        # Indicators of whether maxima are unique114        if self._check_unique:115            max_unique = (torch.sum(max_ps.expand_as(model_ps) == model_ps, 1, keepdim=True) == 1).long()116        else:117            max_unique = torch.ones(target.size(0)).long()118        total_correct = None119        if self._target_indexed:120            total_correct = torch.sum(max_unique.squeeze()*((target == max_index.squeeze()).long()))121        else:122            target_index, has_missing, mask = dist.get_index(target)123            # Count of where model max is same as target124            total_correct = torch.sum(mask*max_unique.squeeze()*((target_index == max_index.squeeze()).long()))125        return float(total_correct)126    def _aggregate_batch(self, agg, batch_result):127        return agg + batch_result128    def _initialize_result(self):129        return 0.0130    def _finalize_result(self, result):131        return result / self._data.get_size()132class ModelStatistic(Evaluation):133    def __init__(self, name, data, data_parameters, stat_fn, trials=1):134        super(ModelStatistic, self).__init__(name, data, data_parameters, trials=trials)135        self._stat_fn = stat_fn136    def _run_once(self, model):137        model.eval()138        stat = self._stat_fn(model)139        model.train()140        return stat141    def _run_batch(self, model, batch):142        pass143    def _aggregate_batch(self, agg, batch_result):144        pass145    def _initialize_result(self):146        pass147    def _finalize_result(self, result):...

writer.py

Source:writer.py

...33        result.rank = test.rank34        result.start_time = start_time35        return result36    @staticmethod37    def _finalize_result(result, end_time):38        result.end_time = end_time39        result.status = "passed" if result.is_successful() else "failed"40    def _lookup_step(self, event):41        try:42            return self.active_steps[event.thread_id]43        except KeyError:44            return None45    def on_test_session_start(self, event):46        self.report.start_time = event.time47    def on_test_session_end(self, event):48        self.report.end_time = event.time49    def on_test_session_setup_start(self, event):50        self.report.test_session_setup = self._initialize_result(event.time)51    def on_test_session_setup_end(self, event):52        self._finalize_result(self.report.test_session_setup, event.time)53    def on_test_session_teardown_start(self, event):54        self.report.test_session_teardown = self._initialize_result(event.time)55    def on_test_session_teardown_end(self, event):56        self._finalize_result(self.report.test_session_teardown, event.time)57    def on_suite_start(self, event):58        suite = event.suite59        suite_result = SuiteResult(suite.name, suite.description)60        suite_result.start_time = event.time61        suite_result.tags.extend(suite.tags)62        suite_result.properties.update(suite.properties)63        suite_result.links.extend(suite.links)64        suite_result.rank = suite.rank65        if suite.parent_suite:66            parent_suite_result = self._get_suite_result(suite.parent_suite)67            parent_suite_result.add_suite(suite_result)68        else:69            self.report.add_suite(suite_result)70    def on_suite_end(self, event):71        suite_result = self._get_suite_result(event.suite)72        suite_result.end_time = event.time73    def on_suite_setup_start(self, event):74        suite_result = self._get_suite_result(event.suite)75        suite_result.suite_setup = self._initialize_result(event.time)76    def on_suite_setup_end(self, event):77        suite_result = self._get_suite_result(event.suite)78        self._finalize_result(suite_result.suite_setup, event.time)79    def on_suite_teardown_start(self, event):80        suite_result = self._get_suite_result(event.suite)81        suite_result.suite_teardown = self._initialize_result(event.time)82    def on_suite_teardown_end(self, event):83        suite_result = self._get_suite_result(event.suite)84        self._finalize_result(suite_result.suite_teardown, event.time)85    def on_test_start(self, event):86        test_result = self._initialize_test_result(event.test, event.time)87        suite_result = self._get_suite_result(event.test.parent_suite)88        suite_result.add_test(test_result)89    def on_test_end(self, event):90        test_result = self._get_test_result(event.test)91        self._finalize_result(test_result, event.time)92    def _bypass_test(self, test, status, status_details, time):93        test_result = self._initialize_test_result(test, time)94        test_result.end_time = time95        test_result.status = status96        test_result.status_details = status_details97        suite_result = self._get_suite_result(test.parent_suite)98        suite_result.add_test(test_result)99    def on_test_skipped(self, event):100        self._bypass_test(event.test, "skipped", event.skipped_reason, event.time)101    def on_test_disabled(self, event):102        self._bypass_test(event.test, "disabled", event.disabled_reason, event.time)103    def on_step_start(self, event):104        result = self.report.get(event.location)105        step = Step(event.step_description)...

main.py

Source:main.py

...73            self.arena.yellow_knight.move(direction=direction)74            self.arena.position_knight_to_arena(self.arena.yellow_knight)75        76    @staticmethod77    def _finalize_result(knight:Knight) -> list:78        item = knight.item79        if item:80            item = str(item)81        return [knight.position, knight.status, item, knight.attack_score, knight.defense_score]82    def finalize_knight_monitor(self):83        results = {}84        results['red'] = self._finalize_result(self.arena.red_knight)85        results['blue'] = self._finalize_result(self.arena.blue_knight)86        results['green'] = self._finalize_result(self.arena.green_knight)87        results['yellow'] = self._finalize_result(self.arena.yellow_knight)88        return results89    def finalize_item_monitor(self) -> None:90        for item_str, item_obj in self.items.items():91            self.item_monitor[item_str] = [item_obj.position, item_obj.equipped]92        return self.item_monitor93if __name__ == '__main__':94    instruction_file = sys.argv[1]95    battle_knights = Main(instruction_file=instruction_file)96    for element, element_details in battle_knights.leaderboard.items():...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.