How to use test_metrics_names method in yandex-tank

Best Python code snippet using yandex-tank

missing_gaps.py

Source:missing_gaps.py

1import numpy as np2import torch3from torch.utils.data import DataLoader4import sys5import pandas as pd6sys.path.append(".")7from scripts.filename import create_file_name8from src.dvip import DVIP_Base9from src.layers_init import init_layers10from utils.plotting_utils import build_plot_name11from utils.process_flags import manage_experiment_configuration12from utils.pytorch_learning import (13    fit,14    predict,15    predict_prior_samples,16    score,17)18args = manage_experiment_configuration()19torch.manual_seed(args.seed)20train_d, train_test_d, test_d = args.dataset.get_split(args.split, args.test_size)21# Get VIP layers22layers = init_layers(train_d.inputs, args.dataset.output_dim, **vars(args))23train_loader = DataLoader(train_d, batch_size=args.batch_size, shuffle=True)24train_test_loader = DataLoader(train_test_d, batch_size=args.batch_size)25test_loader = DataLoader(test_d, batch_size=args.batch_size)26dvip = DVIP_Base(27    args.likelihood,28    layers,29    len(train_d),30    bb_alpha=args.bb_alpha,31    num_samples=args.num_samples_train,32    y_mean=train_d.targets_mean,33    y_std=train_d.targets_std,34    dtype=args.dtype,35    device=args.device,36)37dvip.print_variables()38# Define optimizer and compile model39opt = torch.optim.Adam(dvip.parameters(), lr=args.lr)40# Train the model41losses = fit(42    dvip,43    train_loader,44    opt,45    use_tqdm=True,46    return_loss=True,47    iterations=args.iterations,48    device=args.device,49)50dvip.print_variables()51def get_predictive_results(mean, var):52    prediction_mean = np.mean(mean, axis=0)53    prediction_var = np.mean(var + mean ** 2, axis=0) - prediction_mean ** 254    return prediction_mean, prediction_var55# Change MC samples for test56dvip.num_samples = args.num_samples_test57train_metrics = score(dvip, train_test_loader, args.metrics, device=args.device)58test_metrics = score(dvip, test_loader, args.metrics, device=args.device)59test_metrics_names = list(test_metrics.keys())60num_metrics = len(test_metrics_names)61print("TEST RESULTS: ")62for k, v in test_metrics.items():63    print("\t - {}: {}".format(k, v))64test_mean, test_std = predict(dvip, train_test_loader, device=args.device)65test_prediction_mean, test_prediction_var = get_predictive_results(66    test_mean, test_std ** 267)68dvip.eval()69prior_samples = predict_prior_samples(dvip, train_test_loader).T[0, :, :, -1]70# Create plot title and path71fig_title, path = build_plot_name(**vars(args))72import matplotlib.pyplot as plt73import matplotlib74matplotlib.rcParams["pdf.fonttype"] = 4275matplotlib.rcParams["ps.fonttype"] = 4276f, (ax0, ax1) = plt.subplots(77    2, 1, gridspec_kw={"height_ratios": [3, 1]}, figsize=(16, 9)78)79ax0.scatter(80    train_d.inputs * train_d.inputs_std + train_d.inputs_mean,81    train_d.targets * train_d.targets_std + train_d.targets_mean,82    s=0.2,83    label="Training set",84)85ax0.scatter(86    test_d.inputs * train_d.inputs_std + train_d.inputs_mean,87    test_d.targets,88    s=0.2,89    color="purple",90    label="Test set",91)92X = train_test_d.inputs * train_d.inputs_std + train_d.inputs_mean93sort = np.argsort(X.flatten())94ax0.plot(X[sort], test_prediction_mean[sort], color="orange", label="Predictive mean")95ax0.fill_between(96    X.flatten()[sort],97    (test_prediction_mean - 2 * np.sqrt(test_prediction_var)).flatten()[sort],98    (test_prediction_mean + 2 * np.sqrt(test_prediction_var)).flatten()[sort],99    color="orange",100    alpha=0.3,101    label="Predictive std",102)103ymin, ymax = ax0.get_ylim()104# ax0.plot(X.flatten()[sort], np.mean(prior_samples,axis = 1)[sort], color = "red", alpha = 0.5)105ax0.plot(X.flatten()[sort], prior_samples[sort, :-1], color="red", alpha=0.1)106ax0.plot(107    X.flatten()[sort],108    prior_samples[sort, -1],109    color="red",110    alpha=0.1,111    label="Prior samples",112)113ax0.set_ylim([ymin, ymax])114ax0.tick_params(axis="y", labelsize=24)115ax0.tick_params(axis="x", labelsize=24)116lgnd = ax0.legend(fontsize=24, loc="upper left")117# change the marker size manually for both lines118lgnd.legendHandles[0]._sizes = [40]119lgnd.legendHandles[1]._sizes = [40]120lgnd.legendHandles[4].set_alpha(0.5)121ax1.fill_between(122    X.flatten()[sort],123    np.zeros_like(X.flatten())[sort],124    (np.sqrt(test_prediction_var)).flatten()[sort],125    color="orange",126    alpha=0.3,127    label="Predictive std",128)129ax1.legend(fontsize=24)130ax1.tick_params(axis="y", labelsize=24)131ax1.tick_params(axis="x", labelsize=24)132plt.savefig("plots/extrapolate_" + create_file_name(args) + ".pdf", bbox_inches="tight")133test_metrics_names = list(test_metrics.keys())134num_metrics = len(test_metrics_names)135d = {136    **vars(args),137    **{k + "_train": v for k, v in train_metrics.items()},138    **test_metrics,139}140df = pd.DataFrame.from_dict(d, orient="index").transpose()141df.to_csv(142    path_or_buf="results/extrapolation_" + create_file_name(args) + ".csv",143    encoding="utf-8",...

single_experiment.py

Source:single_experiment.py

1import matplotlib.pyplot as plt2import numpy as np3import pandas as pd4import torch5from torch.utils.data import DataLoader6import sys7sys.path.append(".")8from src.dvip import DVIP_Base9from src.layers_init import init_layers10from utils.process_flags import manage_experiment_configuration11from utils.pytorch_learning import fit_with_metrics, score12from scripts.filename import create_file_name13args = manage_experiment_configuration()14torch.manual_seed(args.seed)15train_dataset, train_test_dataset, test_dataset = args.dataset.get_split(16    args.test_size, args.seed + args.split17)18# Get VIP layers19layers = init_layers(train_dataset.inputs, args.dataset.output_dim, **vars(args))20train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)21train_test_loader = DataLoader(train_test_dataset, batch_size=args.batch_size)22val_loader = DataLoader(test_dataset, batch_size=args.batch_size)23# Create DVIP object24dvip = DVIP_Base(25    args.likelihood,26    layers,27    len(train_dataset),28    bb_alpha=args.bb_alpha,29    num_samples=args.num_samples_train,30    y_mean=train_dataset.targets_mean,31    y_std=train_dataset.targets_std,32    dtype=args.dtype,33    device=args.device,34)35dvip.print_variables()36# Define optimizer and compile model37opt = torch.optim.Adam(dvip.parameters(), lr=args.lr)38dvip.num_samples = args.num_samples_train39# Perform training40train_hist, val_hist = fit_with_metrics(41    dvip,42    train_loader,43    opt,44    args.metrics,45    val_generator=val_loader,46    epochs=args.epochs,47    device=args.device,48)49dvip.print_variables()50# dvip.num_samples = args.num_samples_test51test_metrics = score(dvip, val_loader, args.metrics, device=args.device)52train_metrics = score(dvip, train_test_loader, args.metrics, device=args.device)53test_metrics_names = list(test_metrics.keys())54num_metrics = len(test_metrics_names)55print("TEST RESULTS: ")56for k, v in test_metrics.items():57    print("\t - {}: {}".format(k, v))58df = pd.DataFrame.from_dict(train_hist)59df_val = pd.DataFrame.from_dict(val_hist)60fig = plt.figure(figsize=(20, 10))61ax3 = fig.add_subplot(2, 2, 2)62ax4 = fig.add_subplot(2, 2, 4)63loss = df[["LOSS"]].to_numpy().flatten()64ax3.plot(loss, label="Training loss")65ax3.legend()66ax3.set_title("Loss evolution")67ax4.plot(68    np.arange(loss.shape[0] // 5, loss.shape[0]),69    loss[loss.shape[0] // 5 :],70    label="Training loss",71)72ax4.legend()73ax4.set_title("Loss evolution in last half of epochs")74for i, m in enumerate(test_metrics_names[1:]):75    ax = fig.add_subplot(num_metrics - 1, 2, 2 * i + 1)76    ax.plot(df[[m]].to_numpy(), label="Training {}".format(m))77    ax.plot(df_val[[m]].to_numpy(), label="Validation {}".format(m))78    ymin, ymax = ax.get_ylim()79    d = (ymax - ymin) / 1080    ax.vlines(81        np.argmin(df[[m]].to_numpy()),82        np.min(df[[m]].to_numpy()) - d,83        np.min(df[[m]].to_numpy()) + d,84        color="black",85        label="Minimum value",86    )87    ax.vlines(88        np.argmin(df_val[[m]].to_numpy()),89        np.min(df_val[[m]].to_numpy()) - d,90        np.min(df_val[[m]].to_numpy()) + d,91        color="black",92    )93    ax.legend()94    ax.set_title("{} evolution".format(m))95plt.savefig("plots/" + create_file_name(args) + ".png")96# open file for writing97f = open("plots/" + create_file_name(args) + ".txt", "w")98d = {99    **{k + "_train": v for k, v in train_metrics.items()},100    **test_metrics,101}102# write file103f.write(str(d))104# close file105f.close()106if args.show:...

test_metrics.py

Source:test_metrics.py

...11        yield metrics12    finally:13        metrics.disable()14@pytest.mark.parametrize("namespace", [None, "namespace"])15def test_metrics_names(namespace):16    PI = 3.1415926535917    with mock_metrics(namespace) as metrics:18        client = metrics._client19        m = metrics.get_meter("foo.bar")20        m.increment("my.counter")21        m.distribution("my.dist", PI)22    # The meter gets disabled when the context manager returns, so the client23    # should not have these calls24    m.increment("my.counter2")25    m.distribution("my.dist2", PI)26    assert client.increment.mock_calls == [mock.call("foo.bar.my.counter", 1.0, None)]...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.