How to use create_language_model method in localstack

Best Python code snippet using localstack_python

ngram_language_model.py

Source:ngram_language_model.py

...13    def __init__(self, padded_sentences, ngram_num, language_code) -> None:        14        self.ngram_num = ngram_num15        self.padded_sentences = padded_sentences16        self.vocabulary = self.get_vocab(padded_sentences)17        self.forward_models = [self.create_language_model(n) for n in range(2, ngram_num + 1)]18        self.backward_models = [self.create_language_model(n, backward=True) for n in range(2, ngram_num + 1)]19        20        self.sentence_util = SentenceUtil(language_code)21    22    def select_model(self, seed_words):23        words_num = len(seed_words)24        models_num = len(self.backward_models)25        if words_num <= models_num:26            prev_model = self.backward_models[words_num - 1]27            next_model = self.forward_models[words_num - 1]28        else:29            prev_model = self.backward_models[-1]30            next_model = self.forward_models[-1]31        return prev_model, next_model32    33    def create_language_model(self, N, backward=False):34        ngrams = self.get_ngram(N, self.padded_sentences, backward)35        ngram_language_model = MLE(order=N, vocabulary=self.vocabulary)36        ngram_language_model.fit(ngrams, self.vocabulary)37        return ngram_language_model38    def get_vocab(self, sentences_padded):39        vocab = Vocabulary([word for sent in sentences_padded for word in sent])40        return vocab41        42    def get_ngram(self, N, sentences_padded, backward=False):43        if backward:44            word_ngrams = [ngrams(sent[::-1], N) for sent in sentences_padded]45        else:46            word_ngrams = [ngrams(sent, N) for sent in sentences_padded]47        return word_ngrams...

BigramModel.py

Source:BigramModel.py

...6json_url = os.path.join(SITE_ROOT, "static/data/", "documents_reuters.json")7dict_url = os.path.join(SITE_ROOT, "static/data/", "dictionary_reuters.json")8docs = json.load(open(json_url))9dictionary = json.load(open(dict_url))10def create_language_model():11    all_words = ''12    for values in docs[0].values():13        all_words += values['Title'] + ' ' + values['Body'] + ' '14    freq_words = nltk.FreqDist(all_words.split(' '))15    cfreq_words_2gram = nltk.ConditionalFreqDist(nltk.bigrams(all_words.split(' ')))16    cprob_words_2gram = nltk.ConditionalProbDist(cfreq_words_2gram, nltk.MLEProbDist)17    return cprob_words_2gram18bigram_prob = create_language_model()19def query_expansion(query):20    if len(query.split(' ')) == 1:21        print(bigram_prob[query].samples())22        for key in bigram_prob[query].samples():23            print(bigram_prob[query].prob(key))...

train_lang_models.py

Source:train_lang_models.py

...6except:7    import pickle8from whichlang import cleanup_data, get_input_ngrams, get_ngram_ranks9from whichlang import LANGUAGE_MODELS10def create_language_model(data, language):11    """[summary]12    Args:13        data ([string]): [input data for training]14        language ([type]): [language]15    """16    data = cleanup_data(data)17    input_ngrams = get_input_ngrams(data)18    input_ngram_ranks = get_ngram_ranks(input_ngrams)19    name = '{0}\\{1}.p'.format(LANGUAGE_MODELS, language)20    pickle.dump(input_ngram_ranks, open(name, "wb"))21def main():22    parser = argparse.ArgumentParser()23    parser.add_argument('-f', required=True, help="input file")24    parser.add_argument('-l', required=True, help="language")25    args = parser.parse_args()26    language = args.l27    with open(args.f) as file:28        data = file.read()29    file.close()30    create_language_model(data, language)31    return32if __name__ == "__main__":...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.