Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use test_role_file method in molecule

Best Python code snippet using molecule_python

decompose.py

Source:decompose.py

1from __future__ import unicode_literals, print_function, division2from io import open3import unicodedata4import string5import re6import random7from random import shuffle8import torch9import torch.nn as nn10from torch.autograd import Variable11from torch import optim12import torch.nn.functional as F13import sys14import os15import time16import math17import pickle18import argparse19from sklearn import metrics20from tasks import *21from training import *22from models import *23from evaluation import *24from role_assignment_functions import *25from rolelearner.role_learning_tensor_product_encoder import RoleLearningTensorProductEncoder26import numpy as np27# Code for performing a tensor product decomposition on an28# existing set of vectors29use_cuda = torch.cuda.is_available()30if use_cuda:31    device = torch.device('cuda')32else:33    device = torch.device('cpu')34parser = argparse.ArgumentParser()35parser.add_argument("--data_prefix", help="prefix for the vectors", type=str, default=None)36parser.add_argument("--role_prefix", help="prefix for a file of roles (if used)", type=str,37                    default=None)38parser.add_argument("--role_scheme", help="pre-coded role scheme to use", type=str, default=None)39parser.add_argument("--test_decoder", help="whether to test the decoder (in addition to MSE",40                    type=str, default="False")41parser.add_argument("--decoder", help="decoder type", type=str, default="ltr")42parser.add_argument("--decoder_prefix", help="prefix for the decoder to test", type=str,43                    default=None)44parser.add_argument("--decoder_embedding_size", help="embedding size for decoder", type=int,45                    default=20)46parser.add_argument("--decoder_task", help="task performed by the decoder", type=str,47                    default="auto")48parser.add_argument("--filler_dim", help="embedding dimension for fillers", type=int, default=10)49parser.add_argument("--role_dim", help="embedding dimension for roles", type=int, default=6)50parser.add_argument("--vocab_size", help="vocab size for the training language", type=int,51                    default=10)52parser.add_argument("--hidden_size", help="size of the encodings", type=int, default=60)53parser.add_argument("--save_vectors",54                    help="whether to save vectors generated by the fitted TPR model", type=str,55                    default="False")56parser.add_argument("--save_role_dicts",57                    help="whether to save role_to_index and index_to_role or not", type=str,58                    default="False")59parser.add_argument("--embedding_file", help="file containing pretrained embeddings", type=str,60                    default=None)61parser.add_argument("--unseen_words",62                    help="if using pretrained embeddings: whether to use all zeroes for unseen "63						 "words' embeddings, or to give them random vectors",64                    type=str, default="zero")65parser.add_argument("--extra_test_set", help="additional file to print predictions for", type=str,66                    default=None)67parser.add_argument("--train", help="whether or not to train the model", type=str, default="True")68parser.add_argument("--neighbor_analysis", help="whether to use a neighbor analysis", type=str,69                    default="True")70parser.add_argument("--digits", help="whether this is one of the digit task", type=str,71                    default="True")72parser.add_argument("--final_linear", help="whether to have a final linear layer", type=str,73                    default="True")74parser.add_argument("--embed_squeeze", help="original dimension to be squeezed to filler_dim",75                    type=int, default=None)76parser.add_argument("--role_learning", help="A flag for whether to enable role learning or use "77                                            "the provided roles.", action="store_true")78parser.add_argument("--bidirectional", help="A flag for whether the role learning module is "79                                            "bidirectional.", action="store_true")80parser.add_argument(81    "--role_assignment_shrink_filler_dim",82    help="If specified, the filler embedding is shrunk to this size before being input to the "83         "role assignment LSTM.",84    default=None,85    type=int)86parser.add_argument(87    "--use_one_hot_temperature",88    help="A flag for whether role learning one hot regularization should have an increasing "89         "temperature.",90    action="store_true")91parser.add_argument(92    "--role_assigner_num_layers",93    help="The number of layers for the role assignment network.",94    default=1,95    type=int96)97parser.add_argument(98    "--output_dir",99    help="An optional output folder where files can be saved to.",100    type=str,101    default=None102)103parser.add_argument(104    "--patience",105    help="The number of epochs to train if validation loss isn't improving",106    type=int,107    default=10108)109parser.add_argument(110    "--pretrained_filler_embedding",111    help="A weight file containing a pretrained filler embedding",112    type=str,113    default=None114)115parser.add_argument(116    "--softmax_roles",117    help="Whether the role predictions should be run through a softmax",118    action="store_true"119)120parser.add_argument(121    "--batch_size",122    help="The batch size.",123    default=32,124    type=int125)126parser.add_argument(127    "--one_hot_regularization_weight",128    help="The weight applied to the one hot regularization term",129    type=float,130    default=1.0131)132parser.add_argument(133    "--l2_norm_regularization_weight",134    help="The weight applied to the l2 norm regularization term",135    type=float,136    default=1.0137)138parser.add_argument(139    "--unique_role_regularization_weight",140    help="The weight applied to the unique role regularization term",141    type=float,142    default=1.0143)144parser.add_argument(145    "--num_roles",146    help="The number of roles to give a role learning network. This value is only used when "147         "--role_learning is enabled.",148    type=int,149    default=None150)151parser.add_argument(152    "--data_path",153    help="The location of the data files.",154    type=str,155    default="data"156)157parser.add_argument(158    "--burn_in",159    help="The number of epochs to train without regularization",160    type=int,161    default=0162)163parser.add_argument(164    "--shuffle",165    help="Whether to shuffle the input fillers and corresponding embedding. This is to generate a"166         " value by which to normalize the MSE to compare different role schemes across "167         "different tasks.",168    action="store_true"169)170parser.add_argument(171    "--scan_checkpoint",172    help="The location of a SCAN checkpoint. Settings this argument enables the SCAN task.",173    type=str,174    default=None,175    required=False176)177args = parser.parse_args()178output_dir = None179if args.output_dir:180    output_dir = os.path.join('output/', args.output_dir)181    if not os.path.isdir(output_dir):182        os.makedirs(output_dir)183    with open(os.path.join(output_dir, 'arguments.txt'), 'w') as arguments_file:184        for argument, value in sorted(vars(args).items()):185            arguments_file.write('{}: {}\n'.format(argument, value))186# Create the logfile187if output_dir:188    results_page = open(os.path.join(output_dir, 'log.txt'), 'w')189else:190    if args.final_linear != "True":191        results_page = open(192            "logs/" + args.data_prefix.split("/")[-1] + str(args.role_prefix).split("/")[-1] + str(193                args.role_scheme) + ".filler" + str(args.filler_dim) + ".role" + str(194                args.role_dim) + ".tpr_decomp.nf", "w")195    else:196        results_page = open(197            "logs/" + args.data_prefix.split("/")[-1] + str(args.role_prefix).split("/")[-1] + str(198                args.role_scheme) + ".filler" + str(args.filler_dim) + ".role" + str(199                args.role_dim) + "." + str(args.embed_squeeze) + ".tpr_decomp", "w")200# Load the decoder for computing swapping accuracy201if args.test_decoder == "True" and not args.scan_checkpoint:202    if args.decoder == "ltr":203        decoder = DecoderRNN(args.vocab_size, args.decoder_embedding_size, args.hidden_size)204    elif args.decoder == "bi":205        decoder = DecoderBiRNN(args.vocab_size, args.decoder_embedding_size, args.hidden_size)206    elif args.decoder == "tree":207        decoder = DecoderTreeRNN(args.vocab_size, args.decoder_embedding_size, args.hidden_size)208    else:209        print("Invalid decoder type")210    input_to_output = lambda seq: transform(seq, args.decoder_task)211    if use_cuda:212        decoder.load_state_dict(torch.load("models/decoder_" + args.decoder_prefix + ".weights"))213    else:214        decoder.load_state_dict(215            torch.load("models/decoder_" + args.decoder_prefix + ".weights", map_location='cpu')216        )217    if use_cuda:218        decoder = decoder.cuda()219elif args.test_decoder == "True" and args.scan_checkpoint:220    decoder = SCANDecoderRNN(221        hidden_size=args.hidden_size,222        output_size=args.vocab_size,223        mytype='GRU',224        n_layers=1,225        dropout_p=0226    )227    decoder.eval()228    print('Loading checkpoint: ' + args.scan_checkpoint)229    if use_cuda:230        checkpoint = torch.load(args.scan_checkpoint)231    else:232        checkpoint = torch.load(args.scan_checkpoint, map_location=lambda storage, loc: storage)233    decoder.load_state_dict(checkpoint['decoder_state_dict'])234    if use_cuda:235        decoder = decoder.cuda()236    pairs_test = checkpoint['pairs_test']237    input_to_output_dict = {}238    for input_, output in pairs_test:239        input_to_output_dict[input_] = output240    input_to_output = lambda key: input_to_output_dict[key]241    input_lang = checkpoint['input_lang']242    output_lang = checkpoint['output_lang']243# Prepare the train, dev, and test data244unindexed_train = []245unindexed_dev = []246unindexed_test = []247unindexed_extra = []248filler_to_index = {}249index_to_filler = {}250role_to_index = {}251index_to_role = {}252filler_counter = 0253role_counter = 0254max_length = 0255train_file = open(os.path.join(args.data_path, args.data_prefix + ".data_from_train"), "r")256for line in train_file:257    sequence, vector = line.strip().split("\t")258    unindexed_train.append(([value for value in sequence.split()], Variable(259        torch.FloatTensor(np.array([float(value) for value in vector.split()])))))260    if len(sequence.split()) > max_length:261        max_length = len(sequence.split())262    for filler in sequence.split():263        if filler not in filler_to_index:264            filler_to_index[filler] = filler_counter265            index_to_filler[filler_counter] = filler266            filler_counter += 1267dev_file = open(os.path.join(args.data_path, args.data_prefix + ".data_from_dev"), "r")268for line in dev_file:269    sequence, vector = line.strip().split("\t")270    unindexed_dev.append(([value for value in sequence.split()], Variable(271        torch.FloatTensor(np.array([float(value) for value in vector.split()])))))272    if len(sequence.split()) > max_length:273        max_length = len(sequence.split())274    for filler in sequence.split():275        if filler not in filler_to_index:276            filler_to_index[filler] = filler_counter277            index_to_filler[filler_counter] = filler278            filler_counter += 1279if args.shuffle:280    print("Shuffling the input sequences and corresponding embeddings")281    sequences = []282    embeddings = []283    for sequence, embedding in unindexed_train:284        sequences.append(sequence)285        embeddings.append(embedding)286    shuffle(sequences)287    unindexed_train = []288    for i in range(len(sequences)):289        unindexed_train.append((sequences[i], embeddings[i]))290    sequences = []291    embeddings = []292    for sequence, embedding in unindexed_dev:293        sequences.append(sequence)294        embeddings.append(embedding)295    shuffle(sequences)296    unindexed_dev = []297    for i in range(len(sequences)):298        unindexed_dev.append((sequences[i], embeddings[i]))299test_file = open(os.path.join(args.data_path, args.data_prefix + ".data_from_test"), "r")300for line in test_file:301    sequence, vector = line.strip().split("\t")302    unindexed_test.append(([value for value in sequence.split()], Variable(303        torch.FloatTensor(np.array([float(value) for value in vector.split()])))))304    if len(sequence.split()) > max_length:305        max_length = len(sequence.split())306    for filler in sequence.split():307        if filler not in filler_to_index:308            filler_to_index[filler] = filler_counter309            index_to_filler[filler_counter] = filler310            filler_counter += 1311if args.extra_test_set is not None:312    extra_file = open(os.path.join(args.data_path, args.extra_test_set), "r")313    for line in extra_file:314        sequence, vector = line.strip().split("\t")315        unindexed_extra.append(([value for value in sequence.split()], Variable(316            torch.FloatTensor(np.array([float(value) for value in vector.split()]))).cuda()))317        if len(sequence.split()) > max_length:318            max_length = len(sequence.split())319        for filler in sequence.split():320            if filler not in filler_to_index:321                filler_to_index[filler] = filler_counter322                index_to_filler[filler_counter] = filler323                filler_counter += 1324if args.digits == "True":325    for i in range(10):326        filler_to_index[str(i)] = i327        index_to_filler[i] = str(i)328indexed_train = [([filler_to_index[filler] for filler in elt[0]], elt[1]) for elt in329                 unindexed_train]330indexed_dev = [([filler_to_index[filler] for filler in elt[0]], elt[1]) for elt in unindexed_dev]331indexed_test = [([filler_to_index[filler] for filler in elt[0]], elt[1]) for elt in unindexed_test]332indexed_extra = [([filler_to_index[filler] for filler in elt[0]], elt[1]) for elt in333                 unindexed_extra]334unindexed_train_roles = []335unindexed_dev_roles = []336unindexed_test_roles = []337unindexed_extra_roles = []338n_r = -1339# If there is a file of roles for the fillers, load those roles340if args.role_prefix is not None:341    train_role_file = open(342        os.path.join(args.data_path, args.role_prefix + ".data_from_train.roles"), "r")343    for line in train_role_file:344        unindexed_train_roles.append(line.strip().split())345        for role in line.strip().split():346            if role not in role_to_index:347                role_to_index[role] = role_counter348                index_to_role[role_counter] = role349                role_counter += 1350    dev_role_file = open(351        os.path.join(args.data_path, args.role_prefix + ".data_from_dev.roles"), "r")352    for line in dev_role_file:353        unindexed_dev_roles.append(line.strip().split())354        for role in line.strip().split():355            if role not in role_to_index:356                role_to_index[role] = role_counter357                index_to_role[role_counter] = role358                role_counter += 1359    test_role_file = open(360        os.path.join(args.data_path, args.role_prefix + ".data_from_test.roles"), "r")361    for line in test_role_file:362        unindexed_test_roles.append(line.strip().split())363        for role in line.strip().split():364            if role not in role_to_index:365                role_to_index[role] = role_counter366                index_to_role[role_counter] = role367                role_counter += 1368    if args.extra_test_set is not None:369        extra_role_file = open(os.path.join(args.data_path, args.extra_test_set + ".roles"), "r")370        for line in extra_role_file:371            unindexed_extra_roles.append(line.strip().split())372            for role in line.strip().split():373                if role not in role_to_index:374                    role_to_index[role] = role_counter375                    index_to_role[role_counter] = role376                    role_counter += 1377# Or, if a predefined role scheme is being used, prepare it378elif args.role_scheme is not None:379    if args.role_scheme == "bow":380        n_r, seq_to_roles = create_bow_roles(max_length, len(filler_to_index.keys()))381    elif args.role_scheme == "ltr":382        n_r, seq_to_roles = create_ltr_roles(max_length, len(filler_to_index.keys()))383    elif args.role_scheme == "rtl":384        n_r, seq_to_roles = create_rtl_roles(max_length, len(filler_to_index.keys()))385    elif args.role_scheme == "bi":386        n_r, seq_to_roles = create_bidirectional_roles(max_length, len(filler_to_index.keys()))387    elif args.role_scheme == "wickel":388        n_r, seq_to_roles = create_wickel_roles(max_length, len(filler_to_index.keys()))389    elif args.role_scheme == "tree":390        n_r, seq_to_roles = create_tree_roles(max_length, len(filler_to_index.keys()))391    elif args.role_scheme == "interleave":392        n_r, seq_to_roles = create_interleaving_tree_roles(max_length, len(filler_to_index.keys()))393    else:394        print("Invalid role scheme")395    for pair in indexed_train:396        these_roles = seq_to_roles(pair[0])397        unindexed_train_roles.append(these_roles)398        for role in these_roles:399            if role not in role_to_index:400                role_to_index[role] = role401                index_to_role[role] = role402                role_counter += 1403    for pair in indexed_dev:404        these_roles = seq_to_roles(pair[0])405        unindexed_dev_roles.append(these_roles)406        for role in these_roles:407            if role not in role_to_index:408                role_to_index[role] = role409                index_to_role[role] = role410                role_counter += 1411    for pair in indexed_test:412        these_roles = seq_to_roles(pair[0])413        unindexed_test_roles.append(these_roles)414        for role in these_roles:415            if role not in role_to_index:416                role_to_index[role] = role417                index_to_role[role] = role418                role_counter += 1419    for pair in indexed_extra:420        these_roles = seq_to_roles(pair[0])421        unindexed_extra_roles.append(these_roles)422        for role in these_roles:423            if role not in role_to_index:424                role_to_index[role] = role425                index_to_role[role] = role426                role_counter += 1427else:428    print("No role scheme specified")429indexed_train_roles = [[role_to_index[role] for role in roles] for roles in unindexed_train_roles]430indexed_dev_roles = [[role_to_index[role] for role in roles] for roles in unindexed_dev_roles]431indexed_test_roles = [[role_to_index[role] for role in roles] for roles in unindexed_test_roles]432indexed_extra_roles = [[role_to_index[role] for role in roles] for roles in unindexed_extra_roles]433all_train_data = []434all_dev_data = []435all_test_data = []436all_extra_data = []437if not args.role_learning:438    # Make sure the number of fillers and the number of roles always matches439    for index, element in enumerate(indexed_train):440        if len(element[0]) != len(indexed_train_roles[index]):441            print(index, "ERROR!!!", element[0], indexed_train_roles[index])442        else:443            all_train_data.append((element[0], indexed_train_roles[index], element[1]))444    for index, element in enumerate(indexed_dev):445        if len(element[0]) != len(indexed_dev_roles[index]):446            print(index, "ERROR!!!", element[0], indexed_dev_roles[index])447        else:448            all_dev_data.append((element[0], indexed_dev_roles[index], element[1]))449    for index, element in enumerate(indexed_test):450        if len(element[0]) != len(indexed_test_roles[index]):451            print(index, "ERROR!!!", element[0], indexed_test_roles[index])452        else:453            all_test_data.append((element[0], indexed_test_roles[index], element[1]))454    for index, element in enumerate(indexed_extra):455        if len(element[0]) != len(indexed_extra_roles[index]):456            print(index, "ERROR!!!", element[0], indexed_extra_roles[index])457        else:458            all_extra_data.append((element[0], indexed_extra_roles[index], element[1]))459else:460    # Add dummy roles to the training data, these will be ignored since we are role learning461    for index, element in enumerate(indexed_train):462        all_train_data.append((element[0], 0, element[1]))463    for index, element in enumerate(indexed_dev):464        all_dev_data.append((element[0], 0, element[1]))465    for index, element in enumerate(indexed_test):466        all_test_data.append((element[0], 0, element[1]))467    for index, element in enumerate(indexed_extra):468        all_extra_data.append((element[0], 0, element[1]))469weights_matrix = None470# Prepare the embeddings471# If a file of embeddings was provided, use those.472embedding_dict = None473if args.embedding_file is not None:474    embedding_dict = {}475    embed_file = open(args.embedding_file, "r")476    for line in embed_file:477        parts = line.strip().split()478        if len(parts) == args.filler_dim + 1:479            embedding_dict[parts[0]] = list(map(lambda x: float(x), parts[1:]))480    matrix_len = len(filler_to_index.keys())481    if args.embed_squeeze is not None:482        weights_matrix = np.zeros((matrix_len, args.embed_squeeze))483    else:484        weights_matrix = np.zeros((matrix_len, args.filler_dim))485    for i in range(matrix_len):486        word = index_to_filler[i]487        if word in embedding_dict:488            weights_matrix[i] = embedding_dict[word]489        else:490            if args.unseen_words == "random":491                weights_matrix[i] = np.random.normal(scale=0.6, size=(args.filler_dim,))492            elif args.unseen_words == "zero":493                pass  # It was initialized as zero, so don't need to do anything494            else:495                print("Invalid choice for embeddings of unseen words")496# Initialize the TPDN497if n_r != -1:498    role_counter = n_r499if args.final_linear == "True":500    final_layer_width = args.hidden_size501else:502    final_layer_width = None503if args.role_learning:504    if args.num_roles:505        role_counter = args.num_roles506    print("Using RoleLearningTensorProductEncoder with {} roles".format(role_counter))507    tpr_encoder = RoleLearningTensorProductEncoder(508        n_roles=role_counter,509        n_fillers=filler_counter,510        final_layer_width=final_layer_width,511        filler_dim=args.filler_dim,512        role_dim=args.role_dim,513        pretrained_filler_embeddings=args.pretrained_filler_embedding,514        embedder_squeeze=args.embed_squeeze,515        role_assignment_shrink_filler_dim=args.role_assignment_shrink_filler_dim,516        bidirectional=args.bidirectional,517        num_layers=args.role_assigner_num_layers,518        softmax_roles=args.softmax_roles,519        pretrained_embeddings=weights_matrix,520        one_hot_regularization_weight=args.one_hot_regularization_weight,521        l2_norm_regularization_weight=args.l2_norm_regularization_weight,522        unique_role_regularization_weight=args.unique_role_regularization_weight,523    )524else:525    print("Using TensorProductEncoder")526    tpr_encoder = TensorProductEncoder(527        n_roles=role_counter,528        n_fillers=filler_counter,529        final_layer_width=final_layer_width,530        filler_dim=args.filler_dim,531        role_dim=args.role_dim,532        pretrained_embeddings=weights_matrix,533        embedder_squeeze=args.embed_squeeze,534        pretrained_filler_embeddings=args.pretrained_filler_embedding535    )536if use_cuda:537    tpr_encoder = tpr_encoder.cuda()538#args.data_prefix = args.data_prefix.split("/")[-1] + ".filler" + str(539#    args.filler_dim) + ".role" + str(args.role_dim)540#if args.final_linear != "True":541#    args.data_prefix += ".no_final"542# Train the TPDN543args.role_prefix = str(args.role_prefix).split("/")[-1]544if args.train == "True":545    if output_dir:546        weight_file = os.path.join(output_dir, 'model.tpr')547    else:548        weight_file = "models/" + args.data_prefix + str(549                                  args.role_prefix) + str(args.role_scheme) + ".tpr"550    end_loss = trainIters_tpr(551        all_train_data,552        all_dev_data,553        tpr_encoder,554        n_epochs=1000,555        learning_rate=0.001,556        weight_file=weight_file,557        batch_size=args.batch_size,558        use_one_hot_temperature=args.use_one_hot_temperature,559        patience=args.patience,560        burn_in=args.burn_in561    )562print("Finished training")563# Load the trained TPDN564tpr_encoder.load_state_dict(torch.load(weight_file, map_location=device))565# Prepare test data566all_test_data_orig = all_test_data567all_test_data = batchify_tpr(all_test_data, 1)568test_data_sets = [(Variable(torch.LongTensor([item[0] for item in batch])),569                   Variable(torch.LongTensor([item[1] for item in batch])),570                   torch.cat([item[2].unsqueeze(0).unsqueeze(0) for item in batch], 1)) for571                  batch in all_test_data]572neighbor_counter = 0573neighbor_total_rank = 0574neighbor_correct = 0575# Evaluate on test set576test_symbolic_mse = 0577test_continuous_mse = 0578test_one_hot_loss = 0579test_l2_loss = 0580test_unique_role_loss = 0581for i in range(len(test_data_sets)):582    input_fillers = test_data_sets[i][0]583    input_roles = test_data_sets[i][1]584    target_variable = test_data_sets[i][2]585    if use_cuda:586        input_fillers = input_fillers.cuda()587        input_roles = input_roles.cuda()588        target_variable = target_variable.cuda()589    if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):590        tpr_encoder.eval()591        encoding, role_predictions = tpr_encoder(input_fillers, input_roles)592        test_symbolic_mse += torch.mean(torch.pow(encoding.data - target_variable.data, 2))593        tpr_encoder.train()594        encoding, role_predictions = tpr_encoder(input_fillers, input_roles)595        test_continuous_mse += torch.mean(torch.pow(encoding.data - target_variable.data, 2))596        batch_one_hot_loss, batch_l2_norm_loss, batch_unique_role_loss = \597            tpr_encoder.get_regularization_loss(role_predictions)598        test_one_hot_loss += batch_one_hot_loss599        test_l2_loss += batch_l2_norm_loss600        test_unique_role_loss += batch_unique_role_loss601    else:602        encoding = tpr_encoder(input_fillers, input_roles)603        test_symbolic_mse += torch.mean(torch.pow(encoding.data - target_variable.data, 2))604test_symbolic_mse = test_symbolic_mse / len(all_test_data)605test_continuous_mse = test_continuous_mse / len(all_test_data)606test_one_hot_loss = test_one_hot_loss / len(all_test_data)607test_l2_loss = test_l2_loss / len(all_test_data)608test_unique_role_loss = test_unique_role_loss / len(all_test_data)609total_symbolic_test_loss = \610    test_symbolic_mse + test_one_hot_loss + test_l2_loss + test_unique_role_loss611total_continuous_test_loss = \612    test_continuous_mse + test_one_hot_loss + test_l2_loss + test_unique_role_loss613if args.output_dir:614    results_page.write(output_dir + "\n")615if args.test_decoder == "True" and not args.scan_checkpoint:616    if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):617        tpr_encoder.eval()618    correct, total = score2(tpr_encoder, decoder, input_to_output, batchify(all_test_data, 1),619                            index_to_filler)620    results_page.write(args.data_prefix + str(args.role_prefix) + str(args.role_scheme) +621                       ".tpr" + " Discrete Swapping encoder performance: " + str(correct)622                       + " " + str(total) + "\n")623    if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):624        tpr_encoder.train()625        correct, total = score2(tpr_encoder, decoder, input_to_output, batchify(all_test_data, 1),626                                index_to_filler)627        results_page.write(args.data_prefix + str(args.role_prefix) + str(args.role_scheme) +628                           ".tpr" + " Continuous Swapping encoder performance: " + str(correct)629                           + " " + str(total) + "\n")630elif args.test_decoder == "True" and args.scan_checkpoint:631    if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):632        tpr_encoder.eval()633    correct, total = scoreSCAN(tpr_encoder, decoder, input_to_output, batchify(all_test_data, 1),634                               index_to_filler, output_lang)635    results_page.write('Discrete swapping performance: {} {}\n'.format(correct, total))636    if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):637        tpr_encoder.train()638        correct, total = scoreSCAN(tpr_encoder, decoder, input_to_output, batchify(all_test_data, 1),639                                   index_to_filler, output_lang)640        results_page.write('Continuous swapping performance: {} {}\n'.format(correct, total))641results_page.write('Test symbolic MSE loss: {}\n'.format(test_symbolic_mse))642results_page.write('Test symbolic loss: {}\n'.format(total_symbolic_test_loss.item()))643if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):644    results_page.write('Test continuous MSE loss: {}\n'.format(test_continuous_mse))645    results_page.write('Test continuous loss: {}\n'.format(total_continuous_test_loss.item()))646results_page.write('Test one hot loss: {}\n'.format(test_one_hot_loss))647results_page.write('Test unique role loss: {}\n'.format(test_unique_role_loss))648results_page.write('Test l2 norm loss: {}\n'.format(test_l2_loss))649# Run cluster performance analysis650if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):651    tpr_encoder.train()652    role_assigner = tpr_encoder.role_assigner653    roles_true = []654    roles_predicted = []655    role_indices = torch.tensor([x for x in range(role_counter)], device=device)656    role_embeddings = role_assigner.role_embedding(role_indices)657    num_elements = 0658    num_elements_role_low = 0659    for test_example in all_test_data:660        sequence = test_example[0][0]661        roles_true.extend(test_example[0][1])662        role_emb, role_weights = role_assigner(torch.tensor([sequence], device=device))663        for i in range(len(role_weights)):664            role_prediction = np.argmax(role_weights[i][0].cpu().detach().numpy())665            if role_weights[i][0][role_prediction] < .98:666                num_elements_role_low += 1667            num_elements += 1668            roles_predicted.append(role_prediction)669        # Get the predicted role using cosine distance from the role embeddings670        #for i in range(len(role_emb)):671        #    similarities = []672        #    for j in range(role_counter):673        #        similarities.append(F.cosine_similarity(role_emb[i][0], role_embeddings[j], dim=0))674        #    roles_predicted.append(np.argmax(similarities))675    results_page.write(676        'number of roles used: {}\n'.format(len(np.unique(roles_predicted)))677    )678    results_page.write(679        'percentage low role prediction: {}\n'.format(100 * num_elements_role_low / num_elements)680    )681    results_page.write(682        'adjusted rand index: {}\n'.format(metrics.adjusted_rand_score(roles_true, roles_predicted))683    )684    results_page.write(685        'normalized mutual info: {}\n'.format(metrics.normalized_mutual_info_score(roles_true, roles_predicted))686    )687    results_page.write(688        'adjusted mutual info: {}\n'.format(metrics.adjusted_mutual_info_score(roles_true, roles_predicted))689    )690    results_page.write(691        'homogenity: {}\n'.format(metrics.homogeneity_score(roles_true, roles_predicted))692    )693    results_page.write(694        'completeness: {}\n'.format(metrics.completeness_score(roles_true, roles_predicted))695    )696    results_page.write(697        'v measure score: {}\n'.format(metrics.v_measure_score(roles_true, roles_predicted))698    )699    results_page.write(700        'fowlkes mallows score: {}\n'.format(metrics.fowlkes_mallows_score(roles_true, roles_predicted))701    )702    results_page.write('\n')703# Generate role predictions704if isinstance(tpr_encoder, RoleLearningTensorProductEncoder):705    print('Generating role predictions')706    tpr_encoder.train()707    role_assigner = tpr_encoder.role_assigner708    output_dir = os.path.join('output/', args.output_dir)709    basename = os.path.basename(output_dir)710    train_data_file = open(os.path.join(args.data_path, args.data_prefix + '.data_from_train'), 'r')711    train_role_file = open(712        os.path.join(output_dir, args.data_prefix + '.' + basename + '.data_from_train.roles'), 'w')713    dev_data_file = open(os.path.join(args.data_path, args.data_prefix + '.data_from_dev'), 'r')714    dev_role_file = open(715        os.path.join(output_dir, args.data_prefix + '.' + basename + '.data_from_dev.roles'), 'w')716    test_data_file = open(os.path.join(args.data_path, args.data_prefix + '.data_from_test'), 'r')717    test_role_file = open(718        os.path.join(output_dir, args.data_prefix + '.' + basename + '.data_from_test.roles'), 'w')719    for line in train_data_file:720        sequence, embedding = line.strip().split('\t')721        sequence = sequence.split()722        sequence = list(map(lambda filler: filler_to_index[filler], sequence))723        sequence = torch.LongTensor([sequence])724        if use_cuda:725            sequence = sequence.cuda()726        role_emb, role_weights = role_assigner(sequence)727        roles = []728        for i in range(len(role_weights)):729            roles.append(str(np.argmax(role_weights[i].cpu().detach().numpy())))730        train_role_file.write(' '.join(roles) + '\n')731    train_role_file.close()732    for line in dev_data_file:733        sequence, embedding = line.strip().split("\t")734        sequence = sequence.split()735        sequence = list(map(lambda filler: filler_to_index[filler], sequence))736        sequence = torch.LongTensor([sequence])737        if use_cuda:738            sequence = sequence.cuda()739        role_emb, role_weights = role_assigner(sequence)740        roles = []741        for i in range(len(role_weights)):742            roles.append(str(np.argmax(role_weights[i].cpu().detach().numpy())))743        dev_role_file.write(' '.join(roles) + '\n')744    dev_role_file.close()745    for line in test_data_file:746        sequence, embedding = line.strip().split("\t")747        sequence = sequence.split()748        sequence = list(map(lambda filler: filler_to_index[filler], sequence))749        sequence = torch.LongTensor([sequence])750        if use_cuda:751            sequence = sequence.cuda()752        role_emb, role_weights = role_assigner(sequence)753        roles = []754        for i in range(len(role_weights)):755            roles.append(str(np.argmax(role_weights[i].cpu().detach().numpy())))756        test_role_file.write(' '.join(roles) + '\n')757    test_role_file.close()758# Save the test set predictions, if desired759if args.save_vectors == "True":760    fo_pred = open(761        args.data_prefix + str(args.role_prefix) + str(args.role_scheme) + ".tpr.test_preds", "w")762    for i in range(len(test_data_sets)):763        sequence = all_test_data[i][0]764        pred = tpr_encoder(test_data_sets[i][0], test_data_sets[i][1]).data.cpu().numpy()[0][0]765        sequence = [str(x) for x in sequence]766        pred = [str(x) for x in pred]767        fo_pred.write(" ".join(sequence) + "\t" + " ".join(pred) + "\n")768# Save the role dictionaries, if desired769if args.save_role_dicts == "True":770    with open(args.data_prefix + str(args.role_prefix) + str(771            args.role_scheme) + '.role_to_index.pickle', 'wb') as handle:772        pickle.dump(role_to_index, handle, protocol=pickle.HIGHEST_PROTOCOL)773    with open(args.data_prefix + str(args.role_prefix) + str(774            args.role_scheme) + '.index_to_role.pickle', 'wb') as handle:...

test_ansible_galaxy.py

Source:test_ansible_galaxy.py

...154                                  _instance.options['roles-path'])155    assert not os.path.isdir(role_directory)156    _instance._setup()157    assert os.path.isdir(role_directory)158def test_role_file(role_file, _instance):159    assert role_file == _instance._role_file()160def test_has_requirements_file(_instance):...

test_roles.py

Source:test_roles.py

...160    )161    assert not os.path.isdir(role_directory)162    _instance._setup()163    assert os.path.isdir(role_directory)164def test_role_file(role_file, _instance):165    assert role_file == _instance.requirements_file166def test_has_requirements_file(_instance):...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.