Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use _report_stats method in autotest

Best Python code snippet using autotest_python

report_manager.py

Source:report_manager.py

1""" Report manager utility """2from __future__ import print_function3import time4from datetime import datetime5import onmt6from onmt.utils.logging import logger7def build_report_manager(opt, gpu_rank):8    if opt.tensorboard and gpu_rank == 0:9        from torch.utils.tensorboard import SummaryWriter10        tensorboard_log_dir = opt.tensorboard_log_dir11        if not opt.train_from:12            tensorboard_log_dir += datetime.now().strftime("/%b-%d_%H-%M-%S")13        writer = SummaryWriter(tensorboard_log_dir, comment="Unmt")14    else:15        writer = None16    report_mgr = ReportMgr(opt.report_every, start_time=-1,17                           tensorboard_writer=writer)18    return report_mgr19class ReportMgrBase(object):20    """21    Report Manager Base class22    Inherited classes should override:23        * `_report_training`24        * `_report_step`25    """26    def __init__(self, report_every, start_time=-1.):27        """28        Args:29            report_every(int): Report status every this many sentences30            start_time(float): manually set report start time. Negative values31                means that you will need to set it later or use `start()`32        """33        self.report_every = report_every34        self.start_time = start_time35    def start(self):36        self.start_time = time.time()37    def log(self, *args, **kwargs):38        logger.info(*args, **kwargs)39    def report_training(self, step, num_steps, learning_rate,40                        report_stats, multigpu=False):41        """42        This is the user-defined batch-level traing progress43        report function.44        Args:45            step(int): current step count.46            num_steps(int): total number of batches.47            learning_rate(float): current learning rate.48            report_stats(Statistics): old Statistics instance.49        Returns:50            report_stats(Statistics): updated Statistics instance.51        """52        if self.start_time < 0:53            raise ValueError("""ReportMgr needs to be started54                                (set 'start_time' or use 'start()'""")55        if step % self.report_every == 0:56            if multigpu:57                report_stats = \58                    onmt.utils.Statistics.all_gather_stats(report_stats)59            self._report_training(60                step, num_steps, learning_rate, report_stats)61            return [onmt.utils.Statistics(name=x.name)  for x in report_stats]62        else:63            return report_stats64    def _report_training(self, *args, **kwargs):65        """ To be overridden """66        raise NotImplementedError()67    def report_step(self, lr, step, train_stats=None, valid_stats=None):68        """69        Report stats of a step70        Args:71            train_stats(Statistics): training stats72            valid_stats(Statistics): validation stats73            lr(float): current learning rate74        """75        self._report_step(76            lr, step, train_stats=train_stats, valid_stats=valid_stats)77    def _report_step(self, *args, **kwargs):78        raise NotImplementedError()79class ReportMgr(ReportMgrBase):80    def __init__(self, report_every, start_time=-1., tensorboard_writer=None):81        """82        A report manager that writes statistics on standard output as well as83        (optionally) TensorBoard84        Args:85            report_every(int): Report status every this many sentences86            tensorboard_writer(:obj:`tensorboard.SummaryWriter`):87                The TensorBoard Summary writer to use or None88        """89        super(ReportMgr, self).__init__(report_every, start_time)90        self.tensorboard_writer = tensorboard_writer91    def maybe_log_tensorboard(self, stats, prefix, learning_rate, step):92        if self.tensorboard_writer is not None:93            stats.log_tensorboard(94                prefix, self.tensorboard_writer, learning_rate, step)95    def _report_training(self, step, num_steps, learning_rate,96                         report_stats):97        """98        See base class method `ReportMgrBase.report_training`.99        """100        output_report_stats = []101        for _report_stats in report_stats:102            if _report_stats.n_words == 0:103                continue104            _report_stats.output(step, num_steps,105                                learning_rate, self.start_time)106            # Log the progress using the number of batches on the x-axis.107            self.maybe_log_tensorboard(_report_stats,108                                       "progress_"+_report_stats.name,109                                       learning_rate,110                                       step)111            _report_stats = onmt.utils.Statistics(_report_stats.name)112            output_report_stats.append(_report_stats)113        return output_report_stats114    def _report_step(self, lr, step, train_stats=None, valid_stats=None):115        """116        See base class method `ReportMgrBase.report_step`.117        """118        if train_stats:119            for _train_stats in train_stats:120                self.log('Train name: %s' % _train_stats.name)121                self.log('Train perplexity: %g' % _train_stats.ppl())122                self.log('Train accuracy: %g' % _train_stats.accuracy())123                self.maybe_log_tensorboard(_train_stats,124                                           "train_"+_train_stats.name,125                                           lr,126                                           step)127        if valid_stats:128            for _valid_stats in valid_stats:129                if _valid_stats.n_words == 0:130                    continue131                self.log('Validation name: %s' % _valid_stats.name)132                self.log('Validation perplexity: %g' % _valid_stats.ppl())133                self.log('Validation accuracy: %g' % _valid_stats.accuracy())134                self.maybe_log_tensorboard(_valid_stats,135                                           "valid_"+_valid_stats.name,136                                           lr,...

metrics.py

Source:metrics.py

...42        return r.ok43def run_reporter(stats_key):44    stats = StatHat(stats_key, 'localtunnel.')45    logging.info("starting metrics reporter with {0}".format(stats_key))46    def _report_stats():47        dump = {}48        for m in dump_metrics():49            dump[m['name']] = m['value']50        for metric in monitored_metrics:51            value = dump.get(metric)52            if value:53                if metric.startswith('collect:'):54                    # metrics starting with "collect:" are55                    # counters that will be reset once reported56                    stats.count(metric.split(':')[-1], value)57                    metric_name = metric.split('_count')[0]58                    counter(metric_name).clear()59                else:60                    stats.value(metric, value)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.