Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use cfg_snapshot method in yandex-tank

Best Python code snippet using yandex-tank

retrieval_evaluation.py

Source:retrieval_evaluation.py

1import argparse2import torch3from copy import deepcopy4from pprint import pprint5try:6    from apex.parallel import DistributedDataParallel as DDP7except ImportError:8    pass9from torch.nn.parallel import DistributedDataParallel as torch_DDP10from zerovl.core import init_device, cfg, update_cfg11from zerovl.datasets.clip.clip_dataset import build_torch_valid_loader12from zerovl.models import PIPELINE13from zerovl.utils import build_from_cfg, ENV, logger, all_gather14from zerovl.core.hooks.checkpoint import get_dist_state_dict15from zerovl.tasks.clip.hooks.utils import RetrievalMetric, IndexedEmbInfo16from zerovl.tasks.clip.config import task_cfg_init_fn, update_clip_config17@ENV.root_only18def calcaulate_retrieval_metrics_and_log(collection_dict, cuda_eval = True):19    retrieval = RetrievalMetric()20    index = collection_dict['image_id'] if collection_dict['dataset_name'] != 'imagenet' else collection_dict['caption_id']21    image_embedding = collection_dict['image_embeddings']22    text_embedding = collection_dict['text_embeddings']23    if not cuda_eval:24        index = index.cpu()25        image_embedding = image_embedding.cpu()26        text_embedding = text_embedding.cpu()27    if collection_dict["dataset_name"] != 'imagenet':28        img_emb = IndexedEmbInfo(emb_name='image',group_idx=index,emb_mat=image_embedding).unique()29        text_emb = IndexedEmbInfo(emb_name='text',group_idx=index,emb_mat=text_embedding)30    else:31        img_emb = IndexedEmbInfo(emb_name='image',group_idx=index,emb_mat=image_embedding)32        text_emb = IndexedEmbInfo(emb_name='text',group_idx=index,emb_mat=text_embedding).unique()33    logger.info('{} validation: image emb shape: {}, text emb shape: {}'.format(collection_dict['dataset_name'], img_emb.emb_mat.shape, text_emb.emb_mat.shape))34    i2t = retrieval(img_emb, text_emb)35    t2i = retrieval(text_emb, img_emb)36    37    i2t.update(t2i)38    summary_dict = {}39    for k, v in i2t.items():40        k = k.replace('[image] to [text]', 'I2T')41        k = k.replace('[text] to [image]', 'T2I')42        k = k.replace(': ', '-')43        summary_dict[k] = v * 100.044    summary_dict['RSUM'] = sum(list(summary_dict.values()))45    summary_dict = {'{}_{}'.format(collection_dict['dataset_name'], k): v for k, v in summary_dict.items()}46    logger.emph('-------------- {} Evaluation --------------'.format(collection_dict['dataset_name']))47    pprint(summary_dict)48    logger.emph('-------------- {} Evaluation --------------\n'.format(collection_dict['dataset_name']))49def evaluate_benchmark(loader, model, name):50    collection_keys = ['image_embeddings', 'text_embeddings', 'image_id', 'caption_id']51    epoch_state = {}52    for key in collection_keys:53        epoch_state[key] = []54    for batch in loader:55        batch_dict = {}56        batch_dict['image'], batch_dict['input_ids'], batch_dict['attention_mask'], \57                batch_dict['caption'], batch_dict['image_id'], batch_dict['caption_id'] = batch58        batch_dict = {k: v.cuda(ENV.device, non_blocking=True) for k,v in batch_dict.items() if k not in ['caption']}59        image_embeddings, text_embeddings = model(batch_dict, embeddings='all')60        output = {'image_embeddings': image_embeddings,61                    'text_embeddings': text_embeddings,62                    'image_id': batch_dict['image_id'],63                    'caption_id': batch_dict['caption_id']}64        for key in collection_keys:65            epoch_state[key].append(output[key])66    collection_dict = {}67    for key in collection_keys:68        value = torch.cat(epoch_state[key], 0)69        value = torch.cat(all_gather(value), 0)70        collection_dict[key] = value71    valid_index = collection_dict['image_id'] > -172    collection_dict = {k: v[valid_index] for k,v in collection_dict.items()}73    collection_dict['dataset_name'] = name74    calcaulate_retrieval_metrics_and_log(collection_dict)75def parse_args():76    # Parse args with argparse tool77    parser = argparse.ArgumentParser(description='ZeroVL Evaluation')78    parser.add_argument('--cfg', type=str, required=True,79                        help='experiment configure file name')80    parser.add_argument("--local_rank", type=int, default=0)  # Compatibility with torch launch.py81    parser.add_argument("--ckpt_path", type=str, default='')  82    args, cfg_overrided = parser.parse_known_args()83    # Update config from yaml and argv for override84    update_cfg(task_cfg_init_fn, args.cfg, cfg_overrided, preprocess_fn=update_clip_config)85    # Record the global config and its snapshot (for easy experiment reproduction)86    ENV.cfg = cfg87    ENV.cfg_snapshot = deepcopy(cfg)88    ENV.local_rank = args.local_rank89    return args90def main():91    # Configuration: user config updating and global config generating92    args = parse_args()93    # Initialization: set device, generate global config and inform the user library94    init_device(cfg)95    # Build model96    model = build_from_cfg(cfg.model.name, cfg, PIPELINE).to(ENV.device)97    if cfg.dist.name == 'apex':98        model = DDP(model, delay_allreduce=False)99    elif cfg.dist.name == 'torch':100        model = torch_DDP(model,101                        device_ids=[ENV.local_rank],102                        output_device=ENV.local_rank,103                        find_unused_parameters=False)104    else:105        raise NotImplementedError106    # Runner: building and running107    checkpoint = torch.load(args.ckpt_path, map_location="cpu")108    model_checkpoint = checkpoint['state_dict']  109    model.load_state_dict(get_dist_state_dict(model_checkpoint), strict=False)110    model.eval()111    logger.emph(f'Loaded ckpt path: {args.ckpt_path}')112    for name in cfg.data.valid_name:113        valid_loader = build_torch_valid_loader(cfg, name, mode='valid')114        with torch.no_grad():115            evaluate_benchmark(valid_loader, model, name)116if __name__ == "__main__":...

train.py

Source:train.py

1import argparse2import os3import torch4from copy import deepcopy5from apex.parallel import convert_syncbn_model6from zerovl.core import init_device7from zerovl.datasets import DATALOADER8from zerovl.models import PIPELINE9from zerovl.core import cfg, update_cfg10from zerovl.utils import build_from_cfg, ENV11from zerovl.tasks.linear_prob.linear_runner import LinearProbRunner12from zerovl.tasks.linear_prob.config import task_cfg_init_fn, update_clip_config13def parse_args():14    # Parse args with argparse tool15    parser = argparse.ArgumentParser(description='ZeroVL training')16    parser.add_argument('--cfg', type=str, required=True,17                        help='experiment configure file name')18    parser.add_argument("--local_rank", type=int, default=0)  # Compatibility with torch launch.py19    args, cfg_overrided = parser.parse_known_args()20    # Update config from yaml and argv for override21    update_cfg(task_cfg_init_fn, args.cfg, cfg_overrided, preprocess_fn=update_clip_config)22    # Record the global config and its snapshot (for easy experiment reproduction)23    ENV.cfg = cfg24    ENV.cfg_snapshot = deepcopy(cfg)25    ENV.local_rank = args.local_rank26def main():27    # Configuration: user config updating and global config generating28    parse_args()29    # Initialization: set device, generate global config and inform the user library30    init_device(cfg)31    # Build model32    model = build_from_cfg(cfg.model.name, cfg, PIPELINE).to(ENV.device)33    if cfg.model.syncbn:34        if cfg.dist.name == 'torch':35            model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)36        elif cfg.dist.name == 'apex':37            model = convert_syncbn_model(model)38        else:39            raise NotImplementedError40    41    # Context building: dataloader42    data_loaders = build_from_cfg(cfg.data.name, cfg, DATALOADER)43    # Runner: building and running44    runner = LinearProbRunner(cfg, data_loaders, model)45    runner.run()46if __name__ == '__main__':...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.