How to use is_wiki_word method in Nose

Best Python code snippet using nose

template_odt.py

Source:template_odt.py Github

copy

Full Screen

...757 def _expand_links(self, matches):758 (source, label, external) = self._process_link(matches)759 if(source.startswith("#")):760 temp = source[1:]761 if(self.m_engine.is_wiki_word(temp)):762 return temp763 # DEBUG BRAD: This is a temporary hack to get links of the format -> to work764 source = re.sub(".*?#(.*)", "\\1", source)765 label = re.sub(".*?#(.*)", "\\1", label)766 #print "SOURCE = %s, LABEL = %s" % (source, label)767 #source.replace("->", "")768 #label = label.replace("->", "")769 # Unconvert any XML in case it has already been770 # converted by format_text()771 label = self.unxmlize(label)772 # Now make the label XML safe again773 label = self.xmlize(label)774 if(source[0:4] == "http" or external == True):775 if(source[0:4] != "http"):...

Full Screen

Full Screen

build_union_db.py

Source:build_union_db.py Github

copy

Full Screen

1#! /usr/bin/python32# -*- coding: utf-8 -*-3#--------------------------------------------------------------------------------------------------4# Script to build a union database by merging TSV dictionaries5#6# Usage:7# build_union_db.py [--output str] [--core str] [--gross str] [--top str] [--slim str]8# [--phrase_prob str] [--tran_prob str] [--tran_aux str] [--tran_aux_last str]9# [--rev_prob str] [--cooc_prob str] [--aoa str] [--keyword str] [--min_prob str]10# [--quiet] inputs...11# (An input specified as "label:tsv_file".12#13# Example:14# ./build_union_db.py --output union-body.tkh \15# --phrase_prob enwiki-phrase-prob.tkh --tran_prob tran-prob.tkh \16# --tran_aux dict1.tsv,dict2.tsv --rev_prob jawiki-word-prob.tkh \17# --cooc_prob enwiki-cooc-prob.tkh --min_prob we:0.00001 \18# wj:wiktionary-ja.tsv wn:wordnet.tsv we:wiktionary-en.tsv19#20# Copyright 2020 Google LLC21# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file22# except in compliance with the License. You may obtain a copy of the License at23# https://www.apache.org/licenses/LICENSE-2.024# Unless required by applicable law or agreed to in writing, software distributed under the25# License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,26# either express or implied. See the License for the specific language governing permissions27# and limitations under the License.28#--------------------------------------------------------------------------------------------------29import collections30import json31import logging32import math33import operator34import os35import regex36import sys37import time38import tkrzw39import tkrzw_dict40import tkrzw_pron_util41import tkrzw_tokenizer42import unicodedata43logger = tkrzw_dict.GetLogger()44poses = ("noun", "verb", "adjective", "adverb",45 "pronoun", "auxverb", "preposition", "determiner", "article",46 "interjection", "conjunction", "prefix", "suffix",47 "abbreviation", "phrase", "misc")48inflection_names = ("noun_plural","verb_singular", "verb_present_participle",49 "verb_past", "verb_past_participle",50 "adjective_comparative", "adjective_superlative",51 "adverb_comparative", "adverb_superlative")52etymology_names = ("etymology_prefix", "etymology_core", "etymology_suffix")53top_names = ("pronunciation",) + inflection_names + etymology_names54rel_weights = {"synonym": 1.0,55 "hypernym": 0.9,56 "hyponym": 0.8,57 "antonym": 0.2,58 "derivative": 0.7,59 "relation": 0.5}60noun_suffixes = [61 "es", "s", "ment", "age", "ics", "ness", "ity", "ism", "or", "er", "ist", "t", "pt", "th",62 "ian", "ee", "tion", "sion", "ty", "ance", "ence", "ency", "cy", "ry", "ary", "ery", "ory",63 "al", "age", "dom", "hood", "ship", "nomy", "ing", "ication", "icator", "ce", "se", "son",64 "iation", "ant", "faction", "ture", "sure", "nance", "y", "ess",65]66verb_suffixes = [67 "ify", "en", "ize", "ise", "fy", "ate", "age", "e",68]69adjective_suffixes = [70 "some", "able", "ible", "ic", "ical", "ial", "ive", "ful", "less", "ly", "ous", "y",71 "tic", "ine", "ised", "ing", "ed", "ish", "al", "ual", "icable", "er", "est", "ent", "ific",72 "ative", "tative", "ant", "ary",73]74adverb_suffixes = [75 "ly",76]77particles = {78 "aback", "about", "above", "abroad", "across", "after", "against", "ahead", "along",79 "amid", "among", "apart", "around", "as", "at", "away", "back", "before", "behind",80 "below", "beneath", "between", "beside", "beyond", "by", "despite", "during", "down",81 "except", "for", "forth", "from", "in", "inside", "into", "near", "of", "off", "on",82 "onto", "out", "outside", "over", "per", "re", "since", "than", "through", "throughout",83 "till", "to", "together", "toward", "under", "until", "up", "upon", "with", "within",84 "without", "via",85}86misc_stop_words = {87 "the", "a", "an", "I", "my", "me", "mine", "you", "your", "yours", "he", "his", "him",88 "she", "her", "hers", "it", "its", "they", "their", "them", "theirs",89 "we", "our", "us", "ours", "some", "any", "one", "someone", "something",90 "myself", "yourself", "yourselves", "himself", "herself", "itself", "themselves",91 "who", "whom", "whose", "what", "where", "when", "why", "how", "and", "but", "not", "no",92 "never", "ever", "time", "place", "people", "person", "this", "these", "that", "those",93 "other", "another", "yes", "thou",94 "back", "much", "many", "more", "most", "good", "well", "better", "best", "all",95}96wiki_stop_words = {97 "wikipedia", "encyclopedia", "page", "pages", "edit", "edits", "comment", "comments",98}99no_parents = {100 "number", "ground", "red", "happen", "letter", "monitor", "feed", "winter", "brake",101 "partner", "sister", "environment", "moment", "gun", "shower", "trigger", "wound", "bound",102 "weed", "saw", "copper", "buffer", "lump", "wary", "stove", "doctor", "hinder",103 "tower", "poetry", "parity", "fell", "lay", "bit", "drug", "grass", "shore", "notice",104 "butter", "slang", "grope", "feces", "left", "former", "found", "every", "scheme",105 "evening", "architecture", "hat", "slice", "bite", "tender", "bully", "translate",106 "fence", "liver", "special", "specific", "species", "statistics", "mathematics", "caution",107 "span", "fleet", "language", "gripe", "dribble", "total", "error", "option", "important",108 "shine", "dental", "irony", "transplant", "chemistry", "physics", "grocery", "grade",109 "gutter", "dove", "weary", "queer", "shove", "buggy", "twine", "tier", "rung", "spat",110 "pang", "jibe", "pent", "lode", "gelt", "plant", "plane", "pants", "craze", "grove",111 "downy", "musty", "mangy", "moped", "caper", "balmy", "tinny", "induce", "treaty",112 "chili", "chilli", "chile", "castor", "landry", "start", "baby", "means", "transfer",113 "interior", "exterior", "rabbit", "stripe", "fairy", "shunt", "clove", "abode", "bends",114 "molt", "holler", "feudal", "bounce", "livery", "wan", "sod", "dug", "het", "gat",115 "cover", "book", "cause", "quality", "process", "provide", "entry", "specify", "morning",116 "guarantee", "listen", "identity", "clone", "impress", "belly", "mansion",117}118force_parents = {119 "upwards": "upward", "towards": "toward", "identify": "identity", "guaranty": "guarantee",120 "advice": "advise", "device": "devise", "practice": "practise", "morn": "morning",121 "approximately": "approximate", "invocation": "invoke", "spec": "specify",122 "prisoner": "prison", "emission": "emit", "omission": "omit", "transmission": "transmit",123 "fission": "fissure", "competitive": "compete", "competitor": "compete",124 "conservative": "conserve", "pronunciation": "pronounce", "revelation": "reveal",125 "possession": "possess", "schema": "scheme", "further": "far", "farther": "far",126 "conjunction": "conjunct", "conjunctive": "conjunct", "location": "locate",127 "conjugation": "conjugate", "conjugative": "conjugate", "installation": "install",128 "translation": "translate", "formation": "form", "variation": "vary",129 "importance": "important", "innovative": "innovate", "bated": "bate",130 "chemist": "chemistry", "chemical": "chemistry", "chem": "chemistry",131 "architect": "architecture", "grocer": "grocery", "critic": "critique",132 "chilly": "chill", "launder": "laundry", "tension": "tense", "revolution": "revolve",133 "sensitive": "sense", "mutation": "mutate", "mutant": "mutate", "fated": "fate",134 "apery": "ape", "dingy": "dinge", "precession": "precess", "expertise": "expert",135 "dramatic": "drama", "pic": "picture", "tragic": "tragedy", "manse": "mansion",136 "administrate": "administer", "administrative": "administrate", "inquiry": "inquire",137 "administration": "administrate", "administrator": "administrate", "diplomat": "diplomacy",138 "federal": "federation", "analysis": "analyze", "emphasis": "emphasize",139 "chlorine": "chloride", "recognition": "recognize", "opposite": "oppose", "opponent": "oppose",140 "response": "respond", "tolerant": "tolerate", "remainder": "remain",141 "differential": "different", "differentiate": "different", "failure": "fail",142 "explosive": "explode", "civilization": "civil", "civilize": "civil",143 "success": "succeed", "application": "apply", "therapeutic": "therapy",144 "medical": "medicine", "beneficial": "benefit", "pianist": "piano",145 "blonde": "blond", "classification": "classify", "classify": "class",146 "technique": "technical", "technology": "technique", "technician": "technical",147 "millionaire": "million", "billionaire": "billion", "cigarette": "cigar",148 "adhesion": "adhere", "adhesive": "adhere", "chaotic": "chaos", "disclosure": "disclose",149 "destruction": "destroy", "concession": "concede", "rental": "rent",150 "influential": "influence", "strategic": "strategy", "minimal": "minimum",151 "mini": "minimum", "triangular": "triangle", "rebellion": "rebel",152 "intent": "intend", "replica": "replicate", "timer": "time", "timed": "time",153 "sparkle": "spark", "consensus": "consent", "probably": "probable", "pleasant": "please",154 "philosopher": "philosophy", "radiate": "radius", "tutorial": "tutor",155 "terminal": "terminus", "terminate": "terminus", "grief": "grieve", "grievance": "grieve",156 "anime": "animate", "surgeon": "surgery", "partition": "part", "pretense": "pretend",157 "concept": "conceive", "conceptual": "conceive", "solidarity": "solid",158 "economic": "economy", "economist": "economy", "decisive": "decide",159 "offense": "offend", "offensive": "offend", "necessary": "necessity",160 "frequency": "frequent", "portrait": "portray", "digital": "digit",161 "storage": "store", "nearly": "near", "granny": "grandmother", "sorry": "sorrow",162 "modification": "modify", "characteristic": "character", "anxious": "anxiety",163 "quantify": "quantity", "qualify": "quality", "appendix": "append",164 "quantitative": "quantity", "qualitative": "quality", "supremacy": "supreme",165 "imaginary": "imagine", "imaginative": "imagine", "disastrous": "disaster",166 "systematic": "system", "absorption": "absorb", "disciple": "discipline",167 "favorite": "favor", "prescription": "prescribe", "dominant": "dominate",168 "relief": "relieve", "laughter": "laugh", "participant": "participate",169 "companion": "company", "circular": "circle", "synthetic": "synthesis",170 "choice": "choose", "entrance": "entry", "maintenance": "maintain",171 "social": "society", "substantial": "substance", "identification": "identify",172 "assumption": "assume", "poet": "poetry", "info": "information", "information": "inform",173 "visible": "vision", "realistic": "real", "consumption": "consume", "reception": "recept",174 "photo": "photograph", "demo": "demonstrate", "publish": "public",175 "volunteer": "voluntary", "politician": "politics", "rationale": "rational",176 "physician": "physic", "physicist": "physics", "spectral": "specter",177 "birdie": "bird", "distillate": "distill", "earnings": "earn", "chimp": "chimpanzee",178 "nutrient": "nutrition", "nutritive": "nutrition", "delicacy": "delicate",179 "suspicion": "suspect", "disbelief": "disbelieve", "provocative": "provoke",180 "irritant": "irritate", "displeasure": "displease", "erroneous": "error",181 "humility": "humiliate", "consequence": "consequent", "barbaric": "barbarian",182 "mystic": "mystery", "festive": "festival", "festal": "festival", "intimacy": "intimate",183 "respiratory": "respiration", "respirator": "respiration", "sarcastic": "sarcasm",184 "crucify": "crucifix", "crucifixion": "crucifix", "abdominal": "abdomen",185 "medial": "median", "bureaucrat": "bureau", "wholly": "whole", "consul": "consulate",186 "repetition": "repeat", "repetitive": "repeat", "conquest": "conquer", "cavern": "cave",187 "rubbish": "rubble", "flammable": "flame", "ignorant": "ignore", "solitude": "solitary",188 "curiosity": "curious", "exceptionally": "exceptional", "blotch": "blot", "suckle": "suck",189 "negligent": "neglect", "negligence": "neglect", "infamous": "infamy",190 "deception": "deceit", "deceit": "deceive", "deceptive": "deceive",191 "irritable": "irritate", "prevalent": "prevail", "accusatory": "accuse",192 "conspiracy": "conspire", "envelop": "envelope", "capacitance": "capacitor",193 "romantic": "romance", "perm": "permanent", "feminist": "feminine",194 "demolition": "demolish", "trivial": "trivia", "instantaneous": "instant",195 "expense": "expend", "expenditure": "expend", "memorize": "memory",196 "memo": "memorandum", "consortium": "consort", "medallion": "medal", "godless": "god",197 "abrasion": "abrase", "abrasive": "abrase", "atheist": "atheism", "reunion": "reunite",198 "kindergartner": "kindergarten", "duckling": "duck", "introductory": "introduce",199 "baptism": "baptize", "sled": "sledge", "bobsled": "bobsleigh",200 "tarp": "tarpaulin", "intricacy": "intricate", "reverberate": "reverb",201 "glacial": "glacier", "legislature": "legislate", "redemption": "redeem",202 "predominant": "predominate", "lull": "lullaby", "butt": "buttock", "comfy": "comfort",203 "verification": "verify", "spectacular": "spectacle", "applause": "applaud",204 "theoretical": "theory", "curvature": "curve", "simply": "simple", "cafe": "cafeteria",205 "discussion": "discuss", "comparable": "compare", "comparative": "compare",206 "hysteric": "hysteria", "partial": "part", "generosity": "generous", "maths": "math",207 "prophecy": "prophesy", "prophet": "prophecy", "satisfactory": "satisfy",208 "fulfillment": "fulfill", "sufficient": "suffice", "energetic": "energy",209 "cosmic": "cosmos", "petrol": "petroleum", "applicable": "apply", "splendid": "splendor",210 "reproductive": "reproduce", "apologetic": "apology", "nervous": "nerve",211 "metabolic": "metabolism", "potency": "potent", "impotency": "impotent", "penal": "penalty",212 "migratory": "migrate", "migrant": "migrate", "immigrant": "immigrate", "emigrant": "emigrate",213 "amphibious": "amphibian", "menstrual": "menstruation", "president": "preside",214 "receptionist": "reception", "reception": "receive", "receipt": "receive",215 "receptive": "receive", "remembrance": "remember", "heartbroken": "heartbreak",216 "residential": "residence", "residency": "residence", "resident": "residence",217 "preparatory": "prepare", "glamorous": "glamour", "defense": "defend",218 "cellular": "cell", "viscosity": "viscous", "rhino": "rhinoceros", "hippo": "hippopotamus",219 "ancestral": "ancestor", "negative": "negate", "bacteria": "bacterium",220 "registration": "register", "registry": "register", "inaugural": "inaugurate",221 "alkaline": "alkali", "humane": "human", "divisible": "divide", "capacity": "capable",222 "grandpa": "grandfather", "grandma": "grandmother", "nauseous": "nausea",223 "luncheon": "lunch", "conscientious": "conscience", "mandatory": "mandate",224 "cleric": "clergy", "corrosion": "corrode", "limo": "limousine", "descriptive": "describe",225 "inflammable": "inflame", "inflammation": "inflame", "tremble": "tremor",226 "enthusiast": "enthusiasm", "pussy": "puss", "considerate": "consider",227 "eternity": "eternal", "monstrous": "monster", "clarity": "clarify", "illiteracy": "illiterate",228 "clarification": "clarify", "muscular": "muscle", "furniture": "furnish",229 "perception": "perceive", "percept": "perceive", "sensory": "sense", "symptomatic": "symptom",230 "destination": "destine", "categorical": "category", "ascent": "ascend",231 "ingenuity": "ingenious", "invention": "invent", "gymnast": "gymnastic",232 "propel": "propulsion", "belief": "believe", "whimsy": "whim", "disciplinary": "discipline",233 "mischievous": "mischief", "crazy": "craze", "liquefy": "liquid", "delicacy": "delicate",234 "confectionery": "confection", "resilience": "resilient", "grad": "graduate",235 "therapist": "therapy", "perseverance": "persevere", "intro": "introduction",236 "abolition": "abolish", "reparation": "repair", "testify": "testimony", "sports": "sport",237 "disqualification": "disqualify", "rectangular": "rectangle", "metropolitan": "metropolis",238 "sportsmanship": "sportsman", "atheist": "atheism", "prognostic": "prognosis",239 "assurance": "assure", "insurance": "insure", "extent": "extend", "mineral": "mine",240 "fort": "fortress", "pharmaceutical": "pharmacy", "menstrual": "menstruum",241 "community": "commune", "communal": "commune", "data": "datum", "agenda": "agendum",242 "metric": "meter", "democrat": "democracy", "presumption": "presume", "shelf": "shelve",243 "solitaire": "solitary", "explanatory": "explain", "woolen": "wool", "planar": "plain",244 "northeastern": "northeast", "northwestern": "northwest",245 "southeastern": "southeast", "southwestern": "southwest",246 "eastward": "east", "westward": "west", "wintry": "winter",247}248class BuildUnionDBBatch:249 def __init__(self, input_confs, output_path, core_labels, full_def_labels, gross_labels,250 surfeit_labels, top_labels, slim_labels, tran_list_labels, supplement_labels,251 phrase_prob_path, tran_prob_path, tran_aux_paths, tran_aux_last_paths,252 rev_prob_path, cooc_prob_path, aoa_paths, keyword_path, min_prob_map):253 self.input_confs = input_confs254 self.output_path = output_path255 self.core_labels = core_labels256 self.full_def_labels = full_def_labels257 self.gross_labels = gross_labels258 self.surfeit_labels = surfeit_labels259 self.top_labels = top_labels260 self.slim_labels = slim_labels261 self.tran_list_labels = tran_list_labels262 self.supplement_labels = supplement_labels263 self.phrase_prob_path = phrase_prob_path264 self.tran_prob_path = tran_prob_path265 self.tran_aux_paths = tran_aux_paths266 self.tran_aux_last_paths = tran_aux_last_paths267 self.rev_prob_path = rev_prob_path268 self.cooc_prob_path = cooc_prob_path269 self.aoa_paths = aoa_paths270 self.keyword_path = keyword_path271 self.min_prob_map = min_prob_map272 self.tokenizer = tkrzw_tokenizer.Tokenizer()273 def Run(self):274 start_time = time.time()275 logger.info("Process started: input_confs={}, output_path={}".format(276 str(self.input_confs), self.output_path))277 word_dicts = []278 for label, input_path in self.input_confs:279 slim = label in self.slim_labels280 word_dict = self.ReadInput(input_path, slim)281 word_dicts.append((label, word_dict))282 aux_trans = {}283 for tran_aux_path in self.tran_aux_paths:284 if not tran_aux_path: continue285 self.ReadTranAuxTSV(tran_aux_path, aux_trans)286 aux_last_trans = {}287 for tran_aux_last_path in self.tran_aux_last_paths:288 if not tran_aux_last_path: continue289 self.ReadTranAuxTSV(tran_aux_last_path, aux_last_trans)290 raw_aoa_words = collections.defaultdict(list)291 for aoa_path in self.aoa_paths:292 if not aoa_path: continue293 self.ReadAOAWords(aoa_path, raw_aoa_words)294 aoa_words = {}295 for word, values in raw_aoa_words.items():296 aoa_words[word] = sum(values) / len(values)297 keywords = set()298 if self.keyword_path:299 self.ReadKeywords(self.keyword_path, keywords)300 self.SaveWords(word_dicts, aux_trans, aux_last_trans, aoa_words, keywords)301 logger.info("Process done: elapsed_time={:.2f}s".format(time.time() - start_time))302 def NormalizeText(self, text):303 text = unicodedata.normalize('NFKC', text)304 text = regex.sub(r"[\u2018\u2019\u201A\u201B\u2758\u275B\u275C\u275F\uFF02]", "'", text)305 text = regex.sub(r"[\u201C\u201D\u201E\u201F]", '"', text)306 text = regex.sub(307 r"[\u00AD\u02D7\u2010\u2011\u2012\u2013\u2014\u2015\u2043\u2212\u2796\u2E3A\u2E3B" +308 r"\uFE58\uFE63\uFF0D]", "-", text)309 return text310 def ReadInput(self, input_path, slim):311 start_time = time.time()312 logger.info("Reading an input file: input_path={}".format(input_path))313 word_dict = collections.defaultdict(list)314 num_entries = 0315 with open(input_path) as input_file:316 for line in input_file:317 word = ""318 ipa = ""319 sampa = ""320 texts = []321 inflections = {}322 etymologies = {}323 alternatives = []324 mode = ""325 rel_words = {}326 for field in line.strip().split("\t"):327 columns = field.split("=", 1)328 if len(columns) < 2: continue329 name, value = columns330 value = self.NormalizeText(value)331 value = regex.sub(r"[\p{Z}\p{C}]+", " ", value).strip()332 if name == "word":333 word = value334 elif name == "pronunciation_ipa":335 ipa = value336 elif name == "pronunciation_sampa":337 sampa = value338 elif name.startswith("inflection_"):339 name = regex.sub(r"^[a-z]+_", "", name)340 inflections[name] = inflections.get(name) or value341 elif name.startswith("etymology_"):342 etymologies[name] = value343 elif name == "alternative":344 for alt_word in value.split(","):345 alt_word = alt_word.strip()346 if alt_word:347 alternatives.append(alt_word)348 elif name in poses:349 if slim:350 value = regex.sub(r" \[-+\] .*", "", value).strip()351 if value:352 texts.append((name, value))353 elif name in rel_weights:354 rel_words[name] = value355 elif name == "mode":356 mode = value357 if not ipa and sampa:358 ipa = tkrzw_pron_util.SampaToIPA(sampa)359 if not word or len(word) > 48:360 continue361 if ipa or texts or inflections or etymologies or alternatives:362 key = tkrzw_dict.NormalizeWord(word)363 entry = {"word": word}364 if ipa:365 entry["pronunciation"] = ipa366 for name, value in inflections.items():367 entry[name] = value368 for name, value in etymologies.items():369 entry[name] = value370 if alternatives:371 entry["alternative"] = alternatives372 entry["text"] = texts373 for rel_name, rel_value in rel_words.items():374 entry[rel_name] = rel_value375 if mode:376 key += "\t" + mode377 word_dict[key].append(entry)378 num_entries += 1379 if num_entries % 10000 == 0:380 logger.info("Reading an input: num_entries={}".format(num_entries))381 logger.info("Reading an input done: num_entries={}, elapsed_time={:.2f}s".format(382 num_entries, time.time() - start_time))383 return word_dict384 def ReadTranAuxTSV(self, input_path, aux_trans):385 start_time = time.time()386 logger.info("Reading a translation aux file: input_path={}".format(input_path))387 num_entries = 0388 with open(input_path) as input_file:389 for line in input_file:390 fields = line.strip().split("\t")391 if len(fields) < 2: continue392 word = self.NormalizeText(fields[0])393 values = aux_trans.get(word) or []394 uniq_trans = set()395 for tran in fields[1:]:396 tran = self.NormalizeText(tran)397 tran = regex.sub(r"[\p{Ps}\p{Pe}\p{C}]", "", tran)398 tran = regex.sub(r"[\p{Z}\p{C}]+", " ", tran).strip()399 norm_tran = tkrzw_dict.NormalizeWord(tran)400 if not tran or not norm_tran: continue401 if regex.search(r"\p{Latin}.*の.*(形|分詞|級)", tran): continue402 if norm_tran in uniq_trans: continue403 uniq_trans.add(norm_tran)404 values.append(tran)405 aux_trans[word] = values406 num_entries += 1407 if num_entries % 10000 == 0:408 logger.info("Reading a translation aux file: num_entries={}".format(num_entries))409 logger.info("Reading a translation aux file: num_entries={}, elapsed_time={:.2f}s".format(410 num_entries, time.time() - start_time))411 def ReadAOAWords(self, input_path, aoa_words):412 start_time = time.time()413 logger.info("Reading a AOA file: input_path={}".format(input_path))414 num_entries = 0415 with open(input_path) as input_file:416 is_first = True417 for line in input_file:418 if is_first:419 is_first = False420 continue421 fields = line.strip().split(",")422 if len(fields) != 7: continue423 word = self.NormalizeText(fields[0]).strip()424 occur = fields[3]425 mean = fields[4]426 stddev = fields[5]427 if not word or not regex.fullmatch(r"[0-9.]+", mean): continue428 if not regex.fullmatch(r"[.0-9]+", occur): continue429 mean = float(mean)430 if regex.fullmatch(r"[0-9.]+", stddev):431 mean += float(stddev)432 else:433 mean += 3.0434 aoa_words[word].append(mean)435 num_entries += 1436 if num_entries % 10000 == 0:437 logger.info("Reading a AOA file: num_entries={}".format(num_entries))438 logger.info("Reading a translation aux file: num_entries={}, elapsed_time={:.2f}s".format(439 num_entries, time.time() - start_time))440 def ReadKeywords(self, input_path, keywords):441 start_time = time.time()442 logger.info("Reading a keyword file: input_path={}".format(input_path))443 num_entries = 0444 with open(input_path) as input_file:445 for line in input_file:446 keyword = self.NormalizeText(line).strip()447 keywords.add(keyword)448 num_entries += 1449 if num_entries % 10000 == 0:450 logger.info("Reading a keyword file: num_entries={}".format(num_entries))451 logger.info("Reading a translation aux file: num_entries={}, elapsed_time={:.2f}s".format(452 num_entries, time.time() - start_time))453 def SaveWords(self, word_dicts, aux_trans, aux_last_trans, aoa_words, keywords):454 logger.info("Preparing DBMs")455 phrase_prob_dbm = None456 if self.phrase_prob_path:457 phrase_prob_dbm = tkrzw.DBM()458 phrase_prob_dbm.Open(self.phrase_prob_path, False, dbm="HashDBM").OrDie()459 tran_prob_dbm = None460 if self.tran_prob_path:461 tran_prob_dbm = tkrzw.DBM()462 tran_prob_dbm.Open(self.tran_prob_path, False, dbm="HashDBM").OrDie()463 rev_prob_dbm = None464 if self.rev_prob_path:465 rev_prob_dbm = tkrzw.DBM()466 rev_prob_dbm.Open(self.rev_prob_path, False, dbm="HashDBM").OrDie()467 cooc_prob_dbm = None468 if self.cooc_prob_path:469 cooc_prob_dbm = tkrzw.DBM()470 cooc_prob_dbm.Open(self.cooc_prob_path, False, dbm="HashDBM").OrDie()471 start_time = time.time()472 logger.info("Extracting keys")473 keys = set()474 for label, word_dict in word_dicts:475 for key in word_dict.keys():476 if key.find("\t") >= 0: continue477 keys.add(key)478 logger.info("Extracting keys done: num_keys={}, elapsed_time={:.2f}s".format(479 len(keys), time.time() - start_time))480 start_time = time.time()481 logger.info("Indexing stems")482 stem_index = collections.defaultdict(list)483 for label, word_dict in word_dicts:484 if label in self.supplement_labels: continue485 for key in keys:486 for entry in word_dict[key]:487 word = entry["word"]488 if not regex.fullmatch("[a-z]+", word): continue489 stems = self.GetDerivativeStems(label, entry, word_dict, aux_trans, phrase_prob_dbm)490 if stems:491 valid_stems = set()492 for stem in stems:493 if stem in keys:494 stem_index[stem].append(word)495 valid_stems.add(stem)496 if valid_stems:497 entry["stem"] = list(valid_stems.union(set(entry.get("stem") or [])))498 for label, word_dict in word_dicts:499 if label not in self.core_labels: continue500 for key in keys:501 for entry in word_dict[key]:502 word = entry["word"]503 children = stem_index.get(word)504 if children:505 entry["stem_child"] = list(set(children))506 logger.info("Indexing stems done: num_stems={}, elapsed_time={:.2f}s".format(507 len(stem_index), time.time() - start_time))508 start_time = time.time()509 logger.info("Checking POS of words")510 noun_words = set()511 verb_words = set()512 adj_words = set()513 adv_words = set()514 for label, word_dict in word_dicts:515 if label in self.core_labels:516 for key in keys:517 for entry in word_dict[key]:518 word = entry["word"]519 for pos, text in entry["text"]:520 if pos == "noun": noun_words.add(word)521 if pos == "verb": verb_words.add(word)522 if pos == "adjective": adj_words.add(word)523 if pos == "adverb": adv_words.add(word)524 logger.info("Checking POS of words done: elapsed_time={:.2f}s".format(525 time.time() - start_time))526 start_time = time.time()527 logger.info("Indexing base forms")528 extra_word_bases = {}529 for label, word_dict in word_dicts:530 if label not in self.top_labels: continue531 base_index = collections.defaultdict(list)532 core_index = collections.defaultdict(list)533 for key, entries in word_dict.items():534 for entry in entries:535 word = entry["word"]536 if not regex.fullmatch("[a-z]+", word): continue537 if word in verb_words:538 children = set()539 for part_name in ("verb_present_participle", "verb_past_participle"):540 for part in (entry.get(part_name) or "").split(","):541 part = part.strip()542 if part and part != word and (part in noun_words or part in adj_words):543 base_index[part].append(word)544 extra_word_bases[part] = word545 children.add(part)546 if children:547 entry["base_child"] = list(children)548 if word in adj_words:549 children = set()550 for part_name in ("adjective_comparative", "adjective_superlative"):551 part = entry.get(part_name)552 if part and (part in noun_words or part in adj_words):553 base_index[part].append(word)554 children.add(part)555 if children:556 entry["base_child"] = list(children)557 core = entry.get("etymology_core")558 prefix = entry.get("etymology_prefix")559 suffix = entry.get("etymology_suffix")560 if core and len(core) >= 4 and not prefix and suffix:561 entry["core"] = core562 core_index[core].append(word)563 for key, entries in word_dict.items():564 for entry in entries:565 word = entry["word"]566 if not regex.fullmatch("[a-z]+", word): continue567 bases = base_index.get(word)568 if bases:569 entry["base"] = list(bases)570 children = core_index.get(word)571 if children:572 entry["core_child"] = list(children)573 logger.info("Indexing base forms done: elapsed_time={:.2f}s".format(574 time.time() - start_time))575 start_time = time.time()576 logger.info("Merging entries: num_keys={}".format(len(keys)))577 merged_entries = []578 for key in keys:579 merged_entry = self.MergeRecord(580 key, word_dicts, aux_trans, aoa_words, keywords,581 phrase_prob_dbm, tran_prob_dbm, rev_prob_dbm, cooc_prob_dbm)582 if not merged_entry: continue583 merged_entries.append((key, merged_entry))584 if len(merged_entries) % 1000 == 0:585 logger.info("Merging entries:: num_entries={}".format(len(merged_entries)))586 logger.info("Making records done: num_records={}, elapsed_time={:.2f}s".format(587 len(merged_entries), time.time() - start_time))588 start_time = time.time()589 logger.info("Modifying entries")590 merged_entries = sorted(merged_entries)591 live_words = tkrzw.DBM()592 live_words.Open("", True, dbm="BabyDBM").OrDie()593 rev_live_words = tkrzw.DBM()594 rev_live_words.Open("", True, dbm="BabyDBM").OrDie()595 for key, merged_entry in merged_entries:596 for word_entry in merged_entry:597 word = word_entry["word"]598 prob = float(word_entry.get("probability") or 0)599 value = "{:.8f}".format(prob)600 live_words.Set(word, value).OrDie()601 rev_word = " ".join(reversed(word.split(" ")))602 rev_live_words.Set(rev_word, value).OrDie()603 num_entries = 0604 for key, merged_entry in merged_entries:605 for word_entry in merged_entry:606 word = word_entry["word"]607 entries = []608 for label, word_dict in word_dicts:609 dict_entries = word_dict.get(key)610 if not dict_entries: continue611 for entry in dict_entries:612 if entry["word"] == word:613 entries.append((label, entry))614 self.SetAOA(word_entry, entries, aoa_words, live_words, phrase_prob_dbm)615 self.SetTranslations(word_entry, aux_trans, tran_prob_dbm, rev_prob_dbm)616 self.SetRelations(word_entry, entries, word_dicts, live_words, rev_live_words,617 phrase_prob_dbm, tran_prob_dbm, cooc_prob_dbm, extra_word_bases,618 verb_words, adj_words, adv_words)619 if phrase_prob_dbm and cooc_prob_dbm:620 self.SetCoocurrences(word_entry, entries, word_dicts, phrase_prob_dbm, cooc_prob_dbm)621 num_entries += 1622 if num_entries % 1000 == 0:623 logger.info("Modifying entries: num_records={}".format(num_entries))624 logger.info("Modifying entries done: elapsed_time={:.2f}s".format(time.time() - start_time))625 start_time = time.time()626 logger.info("Finishing entries")627 merged_dict = {}628 for key, merged_entry in merged_entries:629 merged_dict[key] = merged_entry630 num_entries = 0631 for key, merged_entry in merged_entries:632 for word_entry in merged_entry:633 self.CompensateInflections(word_entry, merged_dict, verb_words)634 self.CompensateAlternatives(word_entry, merged_dict)635 self.PropagateTranslations(word_entry, merged_dict, tran_prob_dbm, aux_last_trans)636 num_entries += 1637 if num_entries % 1000 == 0:638 logger.info("Finishing entries R1: num_records={}".format(num_entries))639 num_entries = 0640 for key, merged_entry in merged_entries:641 for word_entry in merged_entry:642 self.SetPhraseTranslations(word_entry, merged_dict, aux_trans, aux_last_trans,643 tran_prob_dbm, phrase_prob_dbm, noun_words, verb_words,644 live_words, rev_live_words)645 self.FilterParents(word_entry, merged_dict)646 self.AbsorbInflections(word_entry, merged_dict)647 num_entries += 1648 if num_entries % 1000 == 0:649 logger.info("Finishing entries R2: num_records={}".format(num_entries))650 logger.info("Finishing entries done: elapsed_time={:.2f}s".format(time.time() - start_time))651 rev_live_words.Close().OrDie()652 live_words.Close().OrDie()653 if cooc_prob_dbm:654 cooc_prob_dbm.Close().OrDie()655 if rev_prob_dbm:656 rev_prob_dbm.Close().OrDie()657 if tran_prob_dbm:658 tran_prob_dbm.Close().OrDie()659 if phrase_prob_dbm:660 phrase_prob_dbm.Close().OrDie()661 start_time = time.time()662 logger.info("Saving records: output_path={}".format(self.output_path))663 word_dbm = tkrzw.DBM()664 num_buckets = len(merged_entries) * 2665 word_dbm.Open(self.output_path, True, dbm="HashDBM", truncate=True,666 align_pow=0, num_buckets=num_buckets)667 num_records = 0668 for key, merged_entry in merged_entries:669 final_entry = []670 for word_entry in merged_entry:671 if word_entry.get("deleted"):672 continue673 for attr_name in list(word_entry.keys()):674 if attr_name.startswith("_"):675 del word_entry[attr_name]676 final_entry.append(word_entry)677 if not final_entry: continue678 serialized = json.dumps(final_entry, separators=(",", ":"), ensure_ascii=False)679 word_dbm.Set(key, serialized)680 num_records += 1681 if num_records % 1000 == 0:682 logger.info("Saving records: num_records={}".format(num_records))683 word_dbm.Close().OrDie()684 logger.info("Saving records done: num_records={}, elapsed_time={:.2f}s".format(685 len(merged_entries), time.time() - start_time))686 def GetDerivativeStems(self, label, entry, word_dict, aux_trans, phrase_prob_dbm):687 word = entry["word"]688 prob = 1.0689 if phrase_prob_dbm:690 prob = self.GetPhraseProb(phrase_prob_dbm, "en", word)691 texts = entry.get("text") or []692 def NormalizeTran(tran):693 tran = tran.strip()694 han_tran = regex.sub(r"[^\p{Han}]", "", tran)695 if len(han_tran) >= 2:696 tran = han_tran697 elif regex.fullmatch(r"\p{Han}\p{Hiragana}+", tran):698 poses = self.tokenizer.GetJaPosList(tran)699 while len(poses) >= 2:700 pos = poses[-1]701 if not regex.fullmatch(r"\p{Hiragana}+", pos[0]): break702 if pos[1] not in ["助詞", "助動詞"] and pos[2] not in ["接尾", "非自立"]: break703 poses = poses[:-1]704 norm_tran = ""705 for pos in poses:706 norm_tran += pos[3]707 if len(norm_tran) >= 2:708 tran = norm_tran709 tran = regex.sub(r"^([\p{Han}]{2,})的$", "\1", tran)710 return tran711 def GetMetadata(in_entry, out_poses, out_deris, out_trans):712 in_word = in_entry["word"]713 for pos, text in in_entry["text"]:714 out_poses.add(pos)715 for part in text.split("[-]"):716 part = part.strip()717 match = regex.search(r"^\[(synonym|derivative)\]: (.*)", part)718 if match:719 expr = regex.sub(r"\[-.*", "", match.group(2))720 for deri in expr.split(","):721 deri = deri.strip()722 if regex.fullmatch("[a-z]+", deri):723 out_deris.add(deri)724 match = regex.search(r"^\[translation\]: (.*)", part)725 if match:726 expr = regex.sub(r"\[-.*", "", match.group(1))727 expr = regex.sub(r"\(.*?\)", "", expr).strip()728 for tran in expr.split(","):729 tran = NormalizeTran(tran)730 if len(tran) >= 2:731 out_trans.add(tran)732 if label in self.gross_labels:733 text = regex.sub(r"\(.*?\)", "", text)734 text = regex.sub(r"（.*?）", "", text)735 for tran in regex.split(r"[,、。]", text):736 tran = NormalizeTran(tran)737 if len(tran) >= 2:738 out_trans.add(tran)739 relation_expr = in_entry.get("relation")740 if relation_expr:741 for rel_word in relation_expr.split(","):742 check_len = max(5, len(rel_word) - 1)743 if rel_word[:check_len] == word[:check_len]:744 out_deris.add(rel_word)745 in_aux_trans = aux_trans.get(in_word)746 if in_aux_trans:747 for tran in in_aux_trans:748 tran = NormalizeTran(tran)749 if len(tran) >= 2:750 out_trans.add(tran)751 poses = set()752 deris = set()753 trans = set()754 GetMetadata(entry, poses, deris, trans)755 deri_probs = {}756 deri_trans = {}757 if phrase_prob_dbm:758 for deri in deris:759 if deri[:3] != word[:3]: continue760 deri_probs[deri] = self.GetPhraseProb(phrase_prob_dbm, "en", deri)761 for deri_entry in word_dict.get(deri) or []:762 if deri_entry["word"] == deri:763 one_deri_poses = set()764 one_deri_deris = set()765 one_deri_trans = set()766 GetMetadata(deri_entry, one_deri_poses, one_deri_deris, one_deri_trans)767 if one_deri_trans:768 deri_trans[deri] = one_deri_trans769 stems = set()770 for pos in poses:771 for rule_pos, suffixes in (772 ("noun", noun_suffixes),773 ("verb", verb_suffixes),774 ("adjective", adjective_suffixes),775 ("adverb", adverb_suffixes)):776 if pos == rule_pos:777 for suffix in suffixes:778 if word.endswith(suffix):779 stem = word[:-len(suffix)]780 if len(stem) >= 2:781 stems.add(stem)782 if len(suffix) >= 2 and stem[-1] == suffix[0]:783 stems.add(stem + suffix[0])784 if len(suffix) >= 2 and stem[-1] == "i":785 stems.add(stem[:-1] + "y")786 if len(suffix) >= 2 and suffix[0] == "i":787 stems.add(stem + "e")788 if len(suffix) >= 2 and suffix[0] == "e":789 stems.add(stem + "e")790 if len(suffix) >= 2 and suffix[0] == "t":791 stems.add(stem + "t")792 if len(suffix) >= 3 and suffix[0] == "s":793 stems.add(stem + "s")794 if suffix == "al" and len(stem) >= 3:795 stems.add(stem + "es")796 stems.add(stem + "e")797 if suffix == "y" and len(stem) >= 3:798 stems.add(stem + "e")799 if suffix in ["tion", "sion"] and len(stem) >= 2:800 stems.add(stem + "e")801 stems.add(stem + "d")802 stems.add(stem + "t")803 stems.add(stem + "s")804 stems.add(stem + "te")805 stems.add(stem + "de")806 stems.add(stem + "se")807 stems.add(stem + "ve")808 if suffix in ["tion", "sion"] and len(stem) >= 3 and stem.endswith("a"):809 stems.add(stem[:-1])810 stems.add(stem[:-1] + "e")811 stems.add(stem[:-1] + "ate")812 if suffix in ["tion", "sion"] and len(stem) >= 3 and stem.endswith("u"):813 stems.add(stem[:-1] + "ve")814 if suffix == "sion" and len(stem) >= 3 and stem.endswith("s"):815 stems.add(stem[:-1] + "t")816 if suffix in ["ible", "able"] and len(stem) >= 2:817 stems.add(stem + "or")818 stems.add(stem + "er")819 stems.add(stem + "ify")820 stems.add(stem + "y")821 if suffix == "ate":822 stems.add(stem + "e")823 if suffix == "al" and len(stem) >= 3 and stem.endswith("r"):824 stems.add(stem[:-1] + "er")825 if suffix == "ive" and len(stem) >= 3 and stem.endswith("s"):826 stems.add(stem[:-1] + "d")827 stems.add(stem[:-1] + "de")828 if suffix == "ic" and len(stem) >= 3:829 stems.add(stem + "y")830 if suffix == "ize" and len(stem) >= 3:831 stems.add(stem + "y")832 if suffix == "ity" and len(stem) >= 6 and stem.endswith("bil"):833 stems.add(stem[:-3] + "ble")834 if suffix == "pt" and len(stem) >= 3:835 stems.add(stem[:-1] + "ve")836 if suffix == "ce" and len(stem) >= 3:837 stems.add(stem + "t")838 stems.add(stem + "d")839 stems.add(stem + "se")840 if suffix == "ian" and len(stem) >= 4:841 stems.add(stem + "y")842 if suffix == "cy" and len(stem) >= 4:843 stems.add(stem + "t")844 if suffix == "faction" and len(stem) >= 4:845 stems.add(stem + "fy")846 if suffix == "ous" and len(stem) >= 4:847 stems.add(stem + "on")848 stems.add(stem + "y")849 stems.add(stem + "e")850 if suffix == "ous" and len(stem) >= 5 and stem.endswith("ul"):851 stems.add(stem[:-2] + "le")852 if suffix == "ant" and len(stem) >= 4:853 stems.add(stem + "ate")854 stems.add(stem + "e")855 if suffix == "ative" and len(stem) >= 4:856 stems.add(stem + "e")857 if suffix in ["er", "or", "ive"] and len(stem) >= 5:858 stems.add(stem + "e")859 if len(stem) >= 3 and stem.endswith("u"):860 stems.add(stem + "e")861 if len(stem) >= 4 and stem.endswith("i"):862 stems.add(stem[:-1] + "e")863 if len(stem) >= 4 and stem.endswith("rr"):864 stems.add(stem[:-1])865 if len(stem) >= 5 and stem.endswith("t"):866 stems.add(stem[:-1] + "ce")867 stems.add(stem[:-1] + "d")868 if len(stem) >= 5 and stem.endswith("v"):869 stems.add(stem + "e")870 if len(stem) >= 8 and stem.endswith("tic"):871 stems.add(stem + "s")872 if len(stem) >= 4 and stem[-1] == stem[-2]:873 stems.add(stem[:-1])874 stems.discard(word)875 #print("STEM", label, word, stems)876 877 valid_stems = set()878 for pos, text in texts:879 match = regex.search(880 r'^[" ]*([\p{Latin}]+)[" ]*の(複数形|三人称|動名詞|現在分詞|過去形|過去分詞)', text)881 if match:882 stem = match.group(1)883 if len(stem) >= 4 and word.startswith(stem):884 valid_stems.add(stem)885 for stem in stems:886 if phrase_prob_dbm:887 stem_prob = self.GetPhraseProb(phrase_prob_dbm, "en", stem)888 else:889 stem_prob = prob890 stem_prob_ratio = stem_prob / prob891 if label not in self.core_labels and stem_prob_ratio < 0.1: continue892 if (stem.find(" ") < 0 and len(stem) >= 8 and len(stem) < len(word) and893 stem_prob_ratio >= 0.5):894 valid_stems.add(stem)895 continue896 stem_entry = None897 for tmp_stem_entry in word_dict.get(stem) or []:898 if tmp_stem_entry["word"] == stem:899 stem_entry = tmp_stem_entry900 is_known = False901 if stem_prob_ratio >= 0.001:902 is_known = True903 if stem_entry and word in (stem_entry.get("related") or []):904 is_known = True905 if len(stem) >= 6 and stem_prob_ratio >= 0.0002:906 is_known = True907 stem_poses = set()908 stem_deris = set()909 stem_trans = set()910 if stem_entry:911 GetMetadata(stem_entry, stem_poses, stem_deris, stem_trans)912 if stem.find(" ") < 0 and len(stem) >= 4 and trans:913 hit_deri = False914 if word in stem_deris:915 hit_deri = True916 hit_tran = False917 for stem_tran in stem_trans:918 if stem_tran in trans:919 hit_tran = True920 if regex.search(r"\p{Han}", stem_tran):921 for tran in trans:922 if tran.find(stem_tran) >= 0 or stem_tran.find(tran) >= 0:923 hit_tran = True924 if ((hit_deri and hit_tran) or (stem_prob_ratio >= 0.1 and hit_deri) or925 (is_known and hit_tran)):926 valid_stems.add(stem)927 check_len = max(3, len(stem) - 2)928 for deri in deris:929 if len(word) < len(deri):930 continue931 deri_prob = deri_probs.get(deri) or 0.0932 deri_prob_ratio = deri_prob / prob933 hit_deri = False934 if deri == stem:935 hit_deri = True936 if stem[:check_len] == deri[:check_len] and len(stem) >= 4:937 prefix = deri[:len(stem)]938 if prefix == stem:939 hit_deri = True940 if len(prefix) >= 6 and tkrzw.Utility.EditDistanceLev(stem, prefix) < 2:941 hit_deri = True942 hit_tran = False943 for deri_tran in deri_trans.get(deri) or []:944 if deri_tran in trans:945 hit_tran = True946 if regex.search(r"\p{Han}", deri_tran):947 for tran in trans:948 if tran.find(deri_tran) >= 0:949 hit_tran = True950 if hit_deri and (deri_prob_ratio >= 0.1 or hit_tran):951 valid_stems.add(deri)952 force_parent = force_parents.get(word)953 if force_parent:954 valid_stems.clear()955 valid_stems.add(force_parent)956 valid_stems.discard(word)957 #print("VALID", word, valid_stems)958 959 return list(valid_stems)960 def MergeRecord(self, key, word_dicts, aux_trans, aoa_words, keywords,961 phrase_prob_dbm, tran_prob_dbm, rev_prob_dbm, cooc_prob_dbm):962 word_entries = {}963 word_shares = collections.defaultdict(float)964 word_trans = collections.defaultdict(set)965 entry_tran_texts = collections.defaultdict(list)966 num_words = 0967 poses = collections.defaultdict(set)968 synonyms = collections.defaultdict(set)969 core = None970 for label, word_dict in word_dicts:971 dict_entries = word_dict.get(key)972 if not dict_entries: continue973 for entry in dict_entries:974 num_words += 1975 word = entry["word"]976 entries = word_entries.get(word) or []977 entries.append((label, entry))978 word_entries[word] = entries979 texts = entry.get("text")980 if texts:981 text_score = len(texts) * 1.0982 for pos, text in texts:983 poses[word].add(pos)984 trans = self.ExtractTextLabelTrans(text)985 if trans:986 text_score += 0.5987 word_trans[word].update(trans)988 word_shares[word] += math.log2(1 + text_score)989 expr = entry.get("synonym")990 if expr:991 for synonym in regex.split(r"[,;]", expr):992 synonym = synonym.strip()993 if regex.search(r"\p{Latin}", synonym) and synonym.lower() != word.lower():994 synonyms[word].add(synonym)995 if not core:996 core = entry.get("core")997 dict_entries = word_dict.get(key + "\ttranslation")998 if dict_entries:999 for entry in dict_entries:1000 word = entry["word"]1001 tran_texts = entry.get("text")1002 if not tran_texts: continue1003 for tran_pos, tran_text in tran_texts:1004 tran_key = word + "\t" + label + "\t" + tran_pos1005 entry_tran_texts[tran_key].append(tran_text)1006 trans = self.ExtractTextLabelTrans(tran_text)1007 if trans:1008 word_trans[word].update(trans)1009 sorted_word_shares = sorted(word_shares.items(), key=lambda x: x[1], reverse=True)1010 if len(sorted_word_shares) > 1 and aux_trans and tran_prob_dbm:1011 spell_ratios = {}1012 if phrase_prob_dbm:1013 word_probs = {}1014 for word, share in sorted_word_shares:1015 if word in word_probs: continue1016 prob = self.GetPhraseProb(phrase_prob_dbm, "en", word)1017 if not regex.search(r"\p{Lu}", word):1018 prob *= 1.11019 word_probs[word] = prob1020 sum_prob = sum([x[1] for x in word_probs.items()])1021 for word, prob in word_probs.items():1022 spell_ratios[word] = prob / sum_prob1023 word_scores = []1024 for word, share in sorted_word_shares:1025 score = 0.01026 if word in keywords:1027 score += 0.11028 cap_aux_trans = aux_trans.get(word) or []1029 if cap_aux_trans:1030 score += 0.11031 cap_word_trans = word_trans.get(word) or []1032 cap_trans = set(cap_aux_trans).union(cap_word_trans)1033 tran_score = 0.01034 if cap_trans:1035 key = tkrzw_dict.NormalizeWord(word)1036 tsv = tran_prob_dbm.GetStr(key)1037 if tsv:1038 fields = tsv.split("\t")1039 max_prob = 0.01040 sum_prob = 0.01041 for i in range(0, len(fields), 3):1042 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])1043 if src != word:1044 prob *= 0.11045 if not regex.search(r"[\p{Han}\{Hiragana}]", trg):1046 prob *= 0.51047 if regex.search(r"\p{Lu}", src):1048 prob *= 0.51049 if trg in cap_trans:1050 max_prob = max(max_prob, prob)1051 sum_prob += prob1052 tran_score += (sum_prob * max_prob) ** 0.51053 spell_score = (spell_ratios.get(word) or 0.0) * 0.51054 score += ((tran_score + 0.05) * (share + 0.05) * (spell_score + 0.05)) ** (1 / 3)1055 word_scores.append((word, score))1056 sorted_word_shares = sorted(word_scores, key=lambda x: x[1], reverse=True)1057 share_sum = sum([x[1] for x in sorted_word_shares])1058 merged_entry = []1059 for word, share in sorted_word_shares:1060 entries = word_entries[word]1061 word_entry = {}1062 word_entry["word"] = word1063 stem = " ".join(self.tokenizer.Tokenize("en", word, False, True))1064 effective_labels = set()1065 surfaces = set([word.lower()])1066 is_keyword = (word in aux_trans or word in aoa_words or word in keywords or1067 (core and core in keywords))1068 word_poses = poses[word]1069 for pos in word_poses:1070 for rule_pos, suffixes in (1071 ("noun", noun_suffixes),1072 ("verb", verb_suffixes),1073 ("adjective", adjective_suffixes),1074 ("adverb", adverb_suffixes)):1075 if pos == rule_pos:1076 for suffix in suffixes:1077 if word.endswith(suffix):1078 pos_stem = word[:-len(suffix)]1079 if len(pos_stem) >= 4:1080 pos_stems = set()1081 pos_stems.add(pos_stem)1082 pos_stems.add(regex.sub(r"i$", r"y", pos_stem))1083 for pos_stem in pos_stems:1084 if pos_stem in aux_trans or pos_stem in aoa_words or pos_stem in keywords:1085 is_keyword = True1086 break1087 if not is_keyword and "verb" in word_poses and regex.fullmatch(r"[a-z ]+", word):1088 tokens = self.tokenizer.Tokenize("en", word, False, False)1089 if len(tokens) >= 2 and tokens[0] in keywords:1090 particle_suffix = True1091 for token in tokens[1:]:1092 if not token in particles:1093 particle_suffix = False1094 break1095 if particle_suffix:1096 is_keyword = True1097 is_super_keyword = is_keyword and bool(regex.fullmatch(r"\p{Latin}{3,}", word))1098 for label, entry in entries:1099 if label not in self.surfeit_labels or is_keyword:1100 effective_labels.add(label)1101 for top_name in top_names:1102 if label not in self.top_labels and top_name in word_entry: continue1103 value = entry.get(top_name)1104 if value:1105 value = unicodedata.normalize('NFKC', value)1106 word_entry[top_name] = value1107 for infl_name in inflection_names:1108 value = entry.get(infl_name)1109 if value:1110 surfaces.add(value.lower())1111 if merged_entry and not effective_labels:1112 continue1113 for label, entry in entries:1114 texts = entry.get("text")1115 if not texts: continue1116 has_good_text = False1117 for pos, text in texts:1118 pure_text = regex.sub(r"[^\p{Latin}\p{Han}\p{Hiragana}\p{Katakana}\d]", "", text)1119 if not pure_text or pure_text == stem:1120 continue1121 has_good_text = True1122 if not has_good_text:1123 continue1124 for pos, text in texts:1125 items = word_entry.get("item") or []1126 tran_key = word + "\t" + label + "\t" + pos1127 sections = []1128 for section in text.split(" [-] "):1129 if not sections:1130 sections.append(section)1131 continue1132 eg_match = regex.search(r"^e\.g\.: (.*)", section)1133 if eg_match:1134 eg_text = eg_match.group(1).lower()1135 eg_words = regex.findall("[-\p{Latin}]+", eg_text)1136 hit = False1137 for surface in surfaces:1138 if surface in eg_words:1139 hit = True1140 break1141 if not hit: continue1142 sections.append(section)1143 text = " [-] ".join(sections)1144 tran_texts = entry_tran_texts.get(tran_key)1145 if tran_texts:1146 del entry_tran_texts[tran_key]1147 for tran_text in tran_texts:1148 tran_item = {"label": label, "pos": pos, "text": tran_text}1149 items.append(tran_item)1150 item = {"label": label, "pos": pos, "text": text}1151 items.append(item)1152 word_entry["item"] = items1153 if "item" not in word_entry:1154 continue1155 num_eff_items = 01156 for item in word_entry["item"]:1157 text = item["text"]1158 if regex.search(r" (of|for) +\"", text) and len(text) < 50:1159 continue1160 if (regex.search(r"\p{Latin}.*の.*(単数|複数|現在|過去|比較|最上).*(形|級|分詞)", text) and1161 len(text) < 30):1162 continue1163 num_eff_items += 11164 if num_eff_items == 0:1165 continue1166 prob = None1167 if phrase_prob_dbm:1168 prob = self.GetPhraseProb(phrase_prob_dbm, "en", word)1169 if stem.lower() != word.lower():1170 if word.endswith("ics"):1171 prob *= 1.11172 elif word.count(" "):1173 prob *= 0.51174 else:1175 prob *= 0.11176 word_entry["probability"] = "{:.7f}".format(prob).replace("0.", ".")1177 if self.min_prob_map:1178 has_good_label = False1179 for item in word_entry["item"]:1180 if item["label"] not in self.min_prob_map:1181 has_good_label = True1182 break1183 if not has_good_label:1184 new_items = []1185 for item in word_entry["item"]:1186 is_good_item = True1187 for label, min_prob in self.min_prob_map.items():1188 if item["label"] == label:1189 if is_keyword:1190 min_prob *= 0.11191 if is_super_keyword:1192 norm_text = tkrzw_dict.NormalizeWord(item["text"])1193 norm_text = regex.sub(r"^(to|a|an|the) +([\p{Latin}])", r"\2", norm_text)1194 dist = tkrzw.Utility.EditDistanceLev(key, norm_text)1195 dist /= max(len(key), len(norm_text))1196 if dist > 0.5 or word in aux_trans or (core and core in aux_trans):1197 min_prob = 0.01198 if prob < min_prob:1199 is_good_item = False1200 if is_good_item:1201 new_items.append(item)1202 word_entry["item"] = new_items1203 if not word_entry.get("item"):1204 continue1205 share_ratio = share / share_sum1206 if share_ratio < 1:1207 word_entry["share"] = "{:.3f}".format(share_ratio).replace("0.", ".")1208 uniq_alternatives = set()1209 scored_alternatives = []1210 for label, entry in entries:1211 alternatives = entry.get("alternative")1212 if alternatives:1213 for alternative in alternatives:1214 norm_alt = tkrzw_dict.NormalizeWord(alternative)1215 if norm_alt == key: continue1216 if label not in self.core_labels:1217 dist = tkrzw.Utility.EditDistanceLev(key, norm_alt)1218 dist_ratio = dist / max(len(key), len(norm_alt))1219 if dist > 4 or dist_ratio > 0.3: continue1220 if alternative not in uniq_alternatives:1221 alt_prob = self.GetPhraseProb(phrase_prob_dbm, "en", alternative)1222 scored_alternatives.append((alternative, alt_prob))1223 uniq_alternatives.add(alternative)1224 if scored_alternatives:1225 scored_alternatives = sorted(scored_alternatives, key=lambda x: x[1], reverse=True)1226 word_entry["alternative"] = [x[0] for x in scored_alternatives]1227 word_synonyms = synonyms[word]1228 if word_synonyms:1229 word_entry["_synonym"] = list(word_synonyms)1230 merged_entry.append(word_entry)1231 return merged_entry1232 def GetPhraseProb(self, prob_dbm, language, word):1233 base_prob = 0.0000000011234 tokens = self.tokenizer.Tokenize(language, word, False, True)1235 if not tokens: return base_prob1236 max_ngram = min(3, len(tokens))1237 fallback_penalty = 1.01238 for ngram in range(max_ngram, 0, -1):1239 if len(tokens) <= ngram:1240 cur_phrase = " ".join(tokens)1241 prob = float(prob_dbm.GetStr(cur_phrase) or 0.0)1242 if prob:1243 return max(prob, base_prob)1244 fallback_penalty *= 0.11245 else:1246 probs = []1247 index = 01248 miss = False1249 while index <= len(tokens) - ngram:1250 cur_phrase = " ".join(tokens[index:index + ngram])1251 cur_prob = float(prob_dbm.GetStr(cur_phrase) or 0.0)1252 if not cur_prob:1253 miss = True1254 break1255 probs.append(cur_prob)1256 index += 11257 if not miss:1258 inv_sum = 01259 for cur_prob in probs:1260 inv_sum += 1 / cur_prob1261 prob = len(probs) / inv_sum1262 prob *= 0.3 ** (len(tokens) - ngram)1263 prob *= fallback_penalty1264 return max(prob, base_prob)1265 fallback_penalty *= 0.11266 return base_prob1267 def SetAOA(self, word_entry, entries, aoa_words, live_words, phrase_prob_dbm):1268 word = word_entry["word"]1269 phrase_prob = min(float(word_entry.get("probability") or 0), 0.0000001)1270 share = float(word_entry.get("share") or 1)1271 share_bias = 0.01272 if share < 0.5:1273 share_bias = (0.5 - share) * 41274 aoa = aoa_words.get(word)1275 if aoa:1276 aoa += share_bias1277 word_entry["aoa"] = "{:.3f}".format(aoa)1278 concepts = set()1279 for label, entry in entries:1280 stems = entry.get("stem")1281 if stems:1282 for stem in stems:1283 concepts.add(stem)1284 core = entry.get("core")1285 if core:1286 concepts.add(core)1287 min_aoa = sys.maxsize1288 for concept in concepts:1289 if not live_words.Get(concept):1290 continue1291 aoa = aoa_words.get(concept)1292 if aoa:1293 if phrase_prob and phrase_prob_dbm:1294 concept_prob = self.GetPhraseProb(phrase_prob_dbm, "en", concept)1295 diff = max(math.log(concept_prob) - math.log(phrase_prob), 0.0)1296 aoa += min(diff * 1.0, 1.0)1297 else:1298 aoa += 1.01299 min_aoa = min(min_aoa, aoa)1300 if min_aoa < sys.maxsize:1301 min_aoa += share_bias1302 word_entry["aoa_concept"] = "{:.3f}".format(min_aoa)1303 bases = set()1304 for label, entry in entries:1305 tmp_bases = entry.get("base")1306 if tmp_bases:1307 for base in tmp_bases:1308 bases.add(base)1309 stem = " ".join(self.tokenizer.Tokenize("en", word, False, True))1310 if stem != word:1311 bases.add(stem)1312 min_aoa = sys.maxsize1313 for base in bases:1314 if not live_words.Get(base):1315 continue1316 aoa = aoa_words.get(base)1317 if aoa:1318 aoa += 1.01319 min_aoa = min(min_aoa, aoa)1320 if min_aoa < sys.maxsize:1321 min_aoa += share_bias1322 word_entry["aoa_base"] = "{:.3f}".format(min_aoa)1323 def ExtractTextLabelTrans(self, text):1324 trans = []1325 match = regex.search(r"\[translation\]: ", text)1326 if match:1327 text = text[match.end():]1328 text = regex.sub(r"\[-.*", "", text)1329 text = regex.sub(r"\(.*?\)", "", text)1330 for tran in text.split(","):1331 tran = unicodedata.normalize('NFKC', tran)1332 tran = tran.strip()1333 tran = regex.sub(r"[\p{S}\p{P}]+ *(が|の|を|に|へ|と|より|から|で|や)", "", tran)1334 tran = regex.sub(r"[～\p{S}\p{P}]", " ", tran)1335 tran = regex.sub(r" +(?=[\p{Han}\p{Hiragana}\p{Katakana}ー])", "", tran)1336 tran = regex.sub(r"[\p{Z}]+", " ", tran).strip()1337 if tran:1338 trans.append(tran)1339 return trans1340 def SetTranslations(self, entry, aux_trans, tran_prob_dbm, rev_prob_dbm):1341 word = entry["word"]1342 tran_probs = {}1343 if tran_prob_dbm:1344 key = tkrzw_dict.NormalizeWord(word)1345 tsv = tran_prob_dbm.GetStr(key)1346 if tsv:1347 fields = tsv.split("\t")1348 extra_records = []1349 for i in range(0, len(fields), 3):1350 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])1351 if regex.search("[っん]$", trg) and self.tokenizer.GetJaLastPos(trg)[1] == "動詞":1352 continue1353 if src != word:1354 prob *= 0.11355 norm_trg = tkrzw_dict.NormalizeWord(trg)1356 if tkrzw_dict.IsStopWord("ja", norm_trg):1357 prob *= 0.71358 elif len(norm_trg) < 2:1359 prob *= 0.91360 prob **= 0.81361 tran_probs[norm_trg] = max(tran_probs.get(norm_trg) or 0.0, prob)1362 stem_trg = regex.sub(1363 r"([\p{Han}\p{Katakana}ー]{2,})(する|すること|される|されること|をする)$",1364 r"\1", norm_trg)1365 if stem_trg != norm_trg:1366 extra_records.append((stem_trg, prob * 0.5))1367 stem_trg = self.tokenizer.CutJaWordNounParticle(norm_trg)1368 if stem_trg != norm_trg:1369 extra_records.append((stem_trg, prob * 0.5))1370 stem_trg = regex.sub(r"([\p{Han}\p{Katakana}ー]{2,})(的|的な|的に)$", r"\1", norm_trg)1371 if stem_trg != norm_trg:1372 extra_records.append((stem_trg, prob * 0.5))1373 if self.tokenizer.IsJaWordSahenNoun(norm_trg):1374 long_trg = norm_trg + "する"1375 extra_records.append((long_trg, prob * 0.5))1376 for extra_trg, extra_prob in extra_records:1377 tran_probs[extra_trg] = max(tran_probs.get(extra_trg) or 0.0, extra_prob)1378 word_aux_trans = aux_trans.get(word)1379 count_aux_trans = {}1380 if word_aux_trans:1381 for aux_tran in word_aux_trans:1382 count = (count_aux_trans.get(aux_tran) or 0) + 11383 count_aux_trans[aux_tran] = count1384 aux_weight = 1.01385 extra_records = []1386 for aux_tran, count in count_aux_trans.items():1387 aux_score = (0.01 ** (1 / (count + 1))) * aux_weight1388 prob = (tran_probs.get(aux_tran) or 0) + aux_score1389 tran_probs[aux_tran] = prob1390 stem_tran = regex.sub(1391 r"([\p{Han}\p{Katakana}ー]{2,})(する|すること|される|されること|をする)$",1392 r"\1", aux_tran)1393 if stem_tran != aux_tran:1394 extra_records.append((stem_tran, aux_score * 0.5))1395 stem_tran = self.tokenizer.CutJaWordNounParticle(aux_tran)1396 if stem_tran != aux_tran:1397 extra_records.append((stem_tran, aux_score * 0.5))1398 stem_tran = regex.sub(r"([\p{Han}\p{Katakana}ー]{2,})(的|的な|的に)$", r"\1", aux_tran)1399 if stem_tran != aux_tran:1400 extra_records.append((stem_tran, aux_score * 0.5))1401 if self.tokenizer.IsJaWordSahenNoun(aux_tran):1402 long_tran = aux_tran + "する"1403 extra_records.append((long_tran, aux_score * 0.5))1404 aux_weight *= 0.91405 for extra_tran, extra_prob in extra_records:1406 tran_probs[extra_tran] = max(tran_probs.get(extra_tran) or 0.0, extra_prob)1407 translations = {}1408 tran_labels = {}1409 def Vote(tran, weight, label):1410 if regex.search(r"^(noun|verb|adj|adv|[0-9])[^\p{Latin}]", tran):1411 return1412 norm_tran = tkrzw_dict.NormalizeWord(tran)1413 score = 0.000011414 if rev_prob_dbm:1415 prob = self.GetPhraseProb(rev_prob_dbm, "ja", tran)1416 prob = max(prob, 0.0000001)1417 prob = math.exp(-abs(math.log(0.001) - math.log(prob))) * 0.11418 if tkrzw_dict.IsStopWord("ja", tran) or tran in ("又は"):1419 prob *= 0.51420 score += prob1421 score *= weight1422 old_score = translations.get(tran) or 0.01423 translations[tran] = max(old_score, score)1424 if label:1425 old_labels = tran_labels.get(norm_tran) or set()1426 old_labels.add(label)1427 tran_labels[norm_tran] = old_labels1428 body_weight = 1.01429 tran_weight = 0.71430 for item in entry["item"]:1431 label = item["label"]1432 pos = item["pos"]1433 sections = item["text"].split(" [-] ")1434 text = sections[0]1435 text = regex.sub(r"。 *(また|または|又は)、.*?。", r"。", text)1436 if (label in self.gross_labels and1437 regex.search(r"[\p{Han}\p{Hiragana}\p{Katakana}ー]", text)):1438 weight = body_weight1439 body_weight *= 0.91440 if regex.search(r"[\(（《〔\{\(]([^)）》〕\}\]]+[・、])?" +1441 r"(俗|俗語|スラング|卑|卑語|隠語|古|古語|廃|廃用|廃語)+[)）》〕\}\]]",1442 text):1443 weight *= 0.11444 text = regex.sub(r"[\(（《〔\{\(].*?[)）》〕\}\]]", "〜", text)1445 text = regex.sub(r"[･・]", "", text)1446 text = regex.sub(r"\p{Z}+", " ", text).strip()1447 if regex.search(1448 r"の(直接法|直説法|仮定法)?(現在|過去)?(第?[一二三]人称)?[ ・･、]?" +1449 r"(単数|複数|現在|過去|比較|最上|進行|完了|動名詞|単純|縮約)+[ ・･、]?" +1450 r"(形|型|分詞|級|動名詞|名詞|動詞|形容詞|副詞)+", text):1451 continue1452 if regex.search(r"の(直接法|直説法|仮定法)(現在|過去)", text):1453 continue1454 if regex.search(r"の(動名詞|異綴|異体|異形|古語|略|省略|短縮|頭字語)", text):1455 continue1456 if regex.search(r"その他、[^。、]{12,}", text):1457 continue1458 text = regex.sub(r" \[-+\] .*", "", text).strip()1459 text = regex.sub(r" -+ .*", "", text).strip()1460 for tran in regex.split("[。|、|；|,|;]", text):1461 if len(translations) > 1:1462 if tran in ("また", "または", "又は", "しばしば"):1463 continue1464 if regex.search(r"^[ \p{Latin}]+〜", tran):1465 continue1466 tran = regex.sub(r"^[\p{S}\p{P}]+ *(が|の|を|に|へ|と|より|から|で|や)", "", tran)1467 tran = regex.sub(r"[～〜]", "", tran)1468 tokens = self.tokenizer.Tokenize("ja", tran, False, False)1469 if len(tokens) > 6:1470 break1471 if regex.search(r"^[ \p{Latin}]+ *など", tran):1472 continue1473 if regex.search(r"[\p{Latin}].*の.*(詞形|綴り)$", tran):1474 continue1475 tran = " ".join(tokens)1476 tran = regex.sub(r"([\p{Han}\p{Hiragana}\p{Katakana}ー]) +", r"\1", tran)1477 tran = regex.sub(r" +([\p{Han}\p{Hiragana}\p{Katakana}ー])", r"\1", tran)1478 tran = regex.sub(r"[\p{Z}]+", " ", tran).strip()1479 if tran:1480 Vote(tran, weight, label)1481 weight *= 0.81482 if label in self.tran_list_labels:1483 for section in sections:1484 trans = self.ExtractTextLabelTrans(section)1485 if not trans: continue1486 weight = tran_weight1487 tran_weight *= 0.91488 uniq_trans = set()1489 for tran in trans:1490 norm_tran = self.tokenizer.NormalizeJaWordForPos(pos, tran)1491 if norm_tran and norm_tran not in uniq_trans:1492 Vote(norm_tran, weight, label)1493 weight *= 0.81494 uniq_trans.add(norm_tran)1495 if label in self.supplement_labels:1496 text = sections[0]1497 uniq_trans = set()1498 for tran in regex.split("[;,]", text):1499 norm_tran = self.tokenizer.NormalizeJaWordForPos(pos, tran.strip())1500 if norm_tran and norm_tran not in uniq_trans:1501 Vote(norm_tran, 0.01, "")1502 pos_scores = {}1503 pos_base_score = 1.01504 for item in entry["item"]:1505 pos = item["pos"]1506 score = pos_base_score1507 if item["label"] not in self.core_labels:1508 score *= 0.751509 pos_scores[pos] = (pos_scores.get(pos) or 0.0) + score1510 pos_base_score *= 0.91511 pos_sum_score = 0.0011512 for pos, score in pos_scores.items():1513 pos_sum_score += score1514 pure_noun = (pos_scores.get("noun") or 0.0) / pos_sum_score >= 0.91515 pure_verb = (pos_scores.get("verb") or 0.0) / pos_sum_score >= 0.91516 pure_adjective = (pos_scores.get("adjective") or 0.0) / pos_sum_score >= 0.91517 pure_adverb = (pos_scores.get("adverb") or 0.0) / pos_sum_score >= 0.91518 bonus_translations = []1519 scored_translations = set()1520 for tran, score in translations.items():1521 tran = unicodedata.normalize('NFKC', tran)1522 norm_tran = tkrzw_dict.NormalizeWord(tran)1523 prob = tran_probs.get(norm_tran)1524 if prob:1525 if len(norm_tran) < 2:1526 prob *= 0.51527 score += prob1528 del tran_probs[norm_tran]1529 scored_translations.add(norm_tran)1530 bonus_translations.append((tran, score))1531 sorted_translations = []1532 for tran, score in bonus_translations:1533 norm_tran = tkrzw_dict.NormalizeWord(tran)1534 if norm_tran not in scored_translations:1535 bonus = 0.01536 for dict_tran, prob in tran_probs.items():1537 if len(dict_tran) >= 2 and norm_tran.startswith(dict_tran):1538 bonus = max(bonus, prob * 0.3)1539 elif len(norm_tran) >= 2 and dict_tran.startswith(norm_tran):1540 bonus = max(bonus, prob * 0.2)1541 elif len(dict_tran) >= 2 and norm_tran.find(dict_tran) >= 0:1542 bonus = max(bonus, prob * 0.1)1543 elif len(norm_tran) >= 2 and dict_tran.find(norm_tran) >= 0:1544 bonus = max(bonus, prob * 0.1)1545 score += bonus1546 if norm_tran in tran_labels:1547 score += (len(tran_labels[norm_tran]) - 1) * 0.0011548 tran_pos = self.tokenizer.GetJaLastPos(tran)1549 if pure_noun:1550 if tran_pos[1] == "名詞" and regex.search(r"\p{Han}", tran):1551 score *= 1.21552 if pure_verb:1553 if tran_pos[1] == "動詞":1554 if regex.search("[うくすつぬふむゆる]$", tran):1555 score *= 1.31556 elif self.tokenizer.IsJaWordSahenNoun(tran):1557 score *= 1.21558 if pure_adjective:1559 tran_pos = self.tokenizer.GetJaLastPos(tran)1560 if tran_pos[1] == "形容詞" or self.tokenizer.IsJaWordAdjvNoun(tran):1561 score *= 1.21562 if (pure_verb or pure_adjective or pure_adverb):1563 if len(tran) <= 1:1564 score *= 0.81565 if regex.search(r"[\p{Katakana}]", tran):1566 score *= 0.71567 if regex.fullmatch(r"[\p{Katakana}ー]+", tran):1568 score *= 0.71569 elif regex.fullmatch(r"[\p{Hiragana}ー]+", tran):1570 score *= 0.91571 elif not regex.search(r"[\p{Han}\p{Hiragana}\p{Katakana}]+", tran):1572 score *= 0.71573 else:1574 if regex.search(r"[\p{Katakana}]", tran):1575 score *= 0.81576 if regex.fullmatch(r"[\p{Katakana}ー]+", tran):1577 score *= 0.81578 elif regex.fullmatch(r"[\p{Hiragana}ー]+", tran):1579 score *= 0.951580 elif not regex.search(r"[\p{Han}\p{Hiragana}\p{Katakana}]+", tran):1581 score *= 0.81582 sorted_translations.append((tran, score))1583 sorted_translations = sorted(sorted_translations, key=lambda x: x[1], reverse=True)1584 deduped_translations = []1585 for tran, score in sorted_translations:1586 norm_tran = tkrzw_dict.NormalizeWord(tran)1587 bias = 1.01588 for prev_tran, prev_score in deduped_translations:1589 if len(prev_tran) >= 2 and norm_tran.startswith(prev_tran):1590 bias = min(bias, 0.4 if len(prev_tran) >= 2 else 0.6)1591 elif len(norm_tran) >= 2 and prev_tran.startswith(norm_tran):1592 bias = min(bias, 0.6 if len(norm_tran) >= 2 else 0.7)1593 elif len(prev_tran) >= 2 and norm_tran.find(prev_tran) >= 0:1594 bias = min(bias, 0.8 if len(prev_tran) >= 2 else 0.9)1595 elif len(norm_tran) >= 2 and prev_tran.find(norm_tran) >= 0:1596 bias = min(bias, 0.8 if len(norm_tran) >= 2 else 0.9)1597 dist = tkrzw.Utility.EditDistanceLev(norm_tran, prev_tran)1598 dist /= max(len(norm_tran), len(prev_tran))1599 if dist < 0.3:1600 bias = min(bias, dist + 0.2)1601 score *= bias1602 deduped_translations.append((tran, score))1603 deduped_translations = sorted(deduped_translations, key=lambda x: x[1], reverse=True)1604 uniq_trans = set()1605 final_translations = []1606 max_elems = int(min(max(math.log2(len(entry["item"])), 2), 8) * 8)1607 for tran, score in deduped_translations:1608 tran = regex.sub(r"^を.*", "", tran)1609 tran = regex.sub(r"・", "", tran)1610 norm_tran = tkrzw_dict.NormalizeWord(tran)1611 if not norm_tran or norm_tran in uniq_trans:1612 continue1613 uniq_trans.add(norm_tran)1614 match = regex.search("(.*)(をする|をやる|する)$", norm_tran)1615 if match:1616 uniq_trans.add(match.group(1) + "する")1617 uniq_trans.add(match.group(1) + "をする")1618 uniq_trans.add(match.group(1) + "をやる")1619 if len(final_translations) < max_elems or score >= 0.001:1620 final_translations.append(tran)1621 sorted_aux_trans = sorted(count_aux_trans.items(), key=lambda x: -x[1])1622 for aux_tran, count in sorted_aux_trans:1623 aux_tran = regex.sub(r"^を.*", "", aux_tran)1624 aux_tran = regex.sub(r"・", "", aux_tran)1625 if pure_noun:1626 aux_tran = self.MakeTranNoun(aux_tran)1627 if pure_verb:1628 aux_tran = self.MakeTranVerb(aux_tran)1629 if pure_adjective:1630 aux_tran = self.MakeTranAdjective(aux_tran)1631 if pure_adverb:1632 aux_tran = self.MakeTranAdverb(aux_tran)1633 if len(final_translations) >= max_elems: break1634 norm_tran = tkrzw_dict.NormalizeWord(aux_tran)1635 if not norm_tran or norm_tran in uniq_trans:1636 continue1637 uniq_trans.add(norm_tran)1638 final_translations.append(aux_tran)1639 if final_translations:1640 entry["translation"] = final_translations1641 def SetRelations(self, word_entry, entries, word_dicts, live_words, rev_live_words,1642 phrase_prob_dbm, tran_prob_dbm, cooc_prob_dbm, extra_word_bases,1643 verb_words, adj_words, adv_words):1644 word = word_entry["word"]1645 norm_word = tkrzw_dict.NormalizeWord(word)1646 scores = {}1647 def Vote(rel_word, label, weight):1648 values = scores.get(rel_word) or []1649 values.append((weight, label))1650 scores[rel_word] = values1651 synonyms = word_entry.get("_synonym")1652 if synonyms:1653 for synonym in synonyms:1654 Vote(synonym, "meta", 0.1)1655 parents = set()1656 children = set()1657 for label, entry in entries:1658 stems = entry.get("stem")1659 if stems:1660 for stem in stems:1661 parents.add(stem)1662 stem_children = entry.get("stem_child")1663 if stem_children:1664 for child in stem_children:1665 children.add(child)1666 core = entry.get("core")1667 if core:1668 parents.add(core)1669 core_children = entry.get("core_child")1670 if core_children:1671 for child in core_children:1672 children.add(child)1673 bases = entry.get("base")1674 if bases:1675 for base in bases:1676 parents.add(base)1677 base_children = entry.get("base_child")1678 if base_children:1679 for child in base_children:1680 children.add(child)1681 for rel_name, rel_weight in rel_weights.items():1682 ent_rel_words = []1683 expr = entry.get(rel_name)1684 if expr:1685 for rel_word in expr.split(","):1686 rel_word = rel_word.strip()1687 ent_rel_words.append(rel_word)1688 if ent_rel_words:1689 scored_rel_words = []1690 for i, rel_word in enumerate(ent_rel_words):1691 weight = 30 / (min(i, 30) + 30)1692 weight *= rel_weight1693 Vote(rel_word, label, weight)1694 texts = entry.get("text")1695 if texts:1696 base_weight = 1.11697 for text in texts:1698 for field in text[1].split(" [-] "):1699 if not field.startswith("[" + rel_name + "]: "): continue1700 field = regex.sub(r"^[^:]+: ", "", field)1701 field = regex.sub(r"\(.*?\) *", "", field)1702 for i, rel_word in enumerate(field.split(",")):1703 rel_word = rel_word.strip()1704 if rel_word:1705 weight = 30 / (min(i, 30) + 30)1706 weight *= rel_weight * base_weight1707 Vote(rel_word, label, weight)1708 base_weight *= 0.951709 extra_word_base = extra_word_bases.get(word)1710 if extra_word_base:1711 parents.add(extra_word_base)1712 alternatives = word_entry.get("alternative")1713 if alternatives:1714 for alternative in alternatives:1715 if word not in force_parents:1716 parents.discard(alternative)1717 if alternative not in force_parents:1718 children.discard(alternative)1719 for variant in self.GetSpellVariants(word):1720 if word not in force_parents:1721 parents.discard(variant)1722 if variant not in force_parents:1723 children.discard(variant)1724 for child in children:1725 parents.discard(child)1726 if word in no_parents:1727 parents.clear()1728 force_parent = force_parents.get(word)1729 if force_parent:1730 parents.clear()1731 parents.add(force_parent)1732 parents = set([x for x in parents if force_parents.get(x) != word])1733 children = set([x for x in children if x not in no_parents])1734 translations = list(word_entry.get("translation") or [])1735 if tran_prob_dbm:1736 tsv = tran_prob_dbm.GetStr(norm_word)1737 if tsv:1738 fields = tsv.split("\t")1739 for i in range(0, len(fields), 3):1740 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])1741 translations.append(trg)1742 translations = set([tkrzw_dict.NormalizeWord(x) for x in translations])1743 rel_words = []1744 for rel_word, votes in scores.items():1745 norm_rel_word = tkrzw_dict.NormalizeWord(rel_word)1746 label_weights = {}1747 for weight, label in votes:1748 old_weight = label_weights.get(label) or 0.01749 label_weights[label] = max(old_weight, weight)1750 total_weight = 01751 for label, weight in label_weights.items():1752 total_weight += weight1753 if tran_prob_dbm:1754 tsv = tran_prob_dbm.GetStr(norm_rel_word)1755 if tsv:1756 bonus = 0.01757 fields = tsv.split("\t")1758 for i in range(0, len(fields), 3):1759 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])1760 norm_tran = tkrzw_dict.NormalizeWord(trg)1761 for dict_tran in translations:1762 if dict_tran == norm_tran:1763 bonus = max(bonus, 1.0)1764 elif len(dict_tran) >= 2 and norm_tran.startswith(dict_tran):1765 bonus = max(bonus, 0.3)1766 elif len(norm_tran) >= 2 and dict_tran.startswith(norm_tran):1767 bonus = max(bonus, 0.2)1768 elif len(dict_tran) >= 2 and norm_tran.find(dict_tran) >= 0:1769 bonus = max(bonus, 0.1)1770 elif len(norm_tran) >= 2 and dict_tran.find(norm_tran) >= 0:1771 bonus = max(bonus, 0.1)1772 dist = tkrzw.Utility.EditDistanceLev(dict_tran, norm_tran)1773 dist /= max(len(dict_tran), len(norm_tran))1774 if dist < 0.3:1775 bonus = max(bonus, 0.3)1776 total_weight += bonus1777 score = 1.01778 if phrase_prob_dbm:1779 prob = self.GetPhraseProb(phrase_prob_dbm, "en", rel_word)1780 prob = max(prob, 0.0000001)1781 score += math.exp(-abs(math.log(0.001) - math.log(prob))) * 0.11782 score *= total_weight1783 if tkrzw_dict.IsStopWord("en", norm_rel_word):1784 if tkrzw_dict.IsStopWord("en", norm_word):1785 score *= 0.31786 else:1787 score *= 0.11788 rel_words.append((rel_word, score))1789 rel_words = sorted(rel_words, key=lambda x: x[1], reverse=True)1790 uniq_words = set()1791 final_rel_words = []1792 for rel_word, score in rel_words:1793 if rel_word in parents or rel_word in children:1794 continue1795 if not live_words.Get(rel_word) or rel_word == word:1796 continue1797 norm_rel_word = tkrzw_dict.NormalizeWord(rel_word)1798 if not norm_rel_word: continue1799 if norm_rel_word in uniq_words: continue1800 uniq_words.add(norm_rel_word)1801 hit = False1802 for label, word_dict in word_dicts:1803 if label in self.surfeit_labels: continue1804 if norm_rel_word in word_dict:1805 hit = True1806 break1807 if not hit: continue1808 final_rel_words.append(rel_word)1809 if final_rel_words:1810 max_elems = int(min(max(math.log2(len(word_entry["item"])), 2), 6) * 6)1811 word_entry["related"] = final_rel_words[:max_elems]1812 scored_parents = []1813 for parent in parents:1814 if not live_words.Get(parent) or parent == word:1815 continue1816 prob = self.GetPhraseProb(phrase_prob_dbm, "en", parent)1817 scored_parents.append((parent, prob))1818 scored_parents = sorted(scored_parents, key=lambda x: x[1], reverse=True)1819 #print("SP", word, scored_parents)1820 1821 if scored_parents:1822 word_entry["parent"] = [x[0] for x in scored_parents]1823 scored_children = []1824 for child in children:1825 if not live_words.Get(child) or child == word or child in parents:1826 continue1827 prob = self.GetPhraseProb(phrase_prob_dbm, "en", child)1828 scored_children.append((child, prob))1829 scored_children = sorted(scored_children, key=lambda x: x[1], reverse=True)1830 if scored_children:1831 word_entry["child"] = [x[0] for x in scored_children]1832 prob = float(live_words.GetStr(word) or 0.0)1833 if prob >= 0.000001 and regex.fullmatch(r"[-\p{Latin}]+", word):1834 prefix = word + " "1835 idioms = []1836 it = live_words.MakeIterator()1837 it.Jump(prefix)1838 while True:1839 rec = it.GetStr()1840 if not rec: break1841 cmp_word, cmp_prob = rec1842 if not cmp_word.startswith(prefix): break1843 cmp_prob = float(cmp_prob)1844 cmp_score = cmp_prob / prob1845 if cmp_score >= 0.001:1846 has_particle = False1847 for cmp_token in cmp_word.split(" ")[1:]:1848 if cmp_token in particles:1849 has_particle = True1850 break1851 if has_particle:1852 cmp_score *= 3.01853 if cmp_word in verb_words or cmp_word in adj_words or cmp_word in adv_words:1854 cmp_score *= 3.01855 idioms.append((cmp_word, cmp_score))1856 it.Next()1857 it = rev_live_words.MakeIterator()1858 it.Jump(prefix)1859 while True:1860 rec = it.GetStr()1861 if not rec: break1862 cmp_word, cmp_prob = rec1863 if not cmp_word.startswith(prefix): break1864 cmp_word = " ".join(reversed(cmp_word.split(" ")))1865 cmp_prob = float(cmp_prob)1866 cmp_score = cmp_prob / prob1867 if cmp_score >= 0.001:1868 has_particle = False1869 for cmp_token in cmp_word.split(" ")[:-1]:1870 if cmp_token in particles:1871 has_particle = True1872 break1873 if has_particle:1874 cmp_score *= 3.01875 if cmp_word in verb_words or cmp_word in adj_words or cmp_word in adv_words:1876 cmp_score *= 3.01877 cmp_score * 0.91878 idioms.append((cmp_word, cmp_score))1879 it.Next()1880 idioms = sorted(idioms, key=lambda x: x[1], reverse=True)1881 uniq_idioms = set()1882 final_idioms = []1883 for idiom, prob in idioms:1884 if idiom in uniq_idioms: continue1885 uniq_idioms.add(idiom)1886 final_idioms.append(idiom)1887 if final_idioms:1888 max_elems = int(min(max(math.log2(len(word_entry["item"])), 2), 6) * 4)1889 word_entry["idiom"] = final_idioms[:max_elems]1890 def SetCoocurrences(self, word_entry, entries, word_dicts, phrase_prob_dbm, cooc_prob_dbm):1891 word = word_entry["word"]1892 norm_word = tkrzw_dict.NormalizeWord(word)1893 tokens = self.tokenizer.Tokenize("en", word, True, True)1894 cooc_words = collections.defaultdict(float)1895 max_word_weight = 0.01896 for token in tokens:1897 phrase_prob = self.GetPhraseProb(phrase_prob_dbm, "en", token)1898 word_idf = math.log(phrase_prob) * -11899 word_weight = word_idf ** 21900 max_word_weight = max(max_word_weight, word_weight)1901 tsv = cooc_prob_dbm.GetStr(token)1902 if tsv:1903 for field in tsv.split("\t")[:32]:1904 cooc_word, cooc_prob = field.split(" ", 1)1905 cooc_tokens = self.tokenizer.Tokenize("en", cooc_word, True, True)1906 for cooc_token in cooc_tokens:1907 if cooc_token and cooc_word not in tokens:1908 cooc_words[cooc_token] += float(cooc_prob) * word_weight1909 def_token_labels = collections.defaultdict(set)1910 for item in word_entry["item"]:1911 label = item["label"]1912 if label not in self.full_def_labels: continue1913 text = item["text"]1914 text = regex.sub(r" \[-.*", "", text).strip()1915 if regex.search(r"^\[-.*", text): continue1916 text = regex.sub(r"\(.*?\)", "", text)1917 text = regex.sub(r"\[.*?\]", "", text)1918 if not text: continue1919 def_tokens = self.tokenizer.Tokenize("en", text, True, True)1920 for def_token in def_tokens:1921 if not regex.fullmatch(r"[\p{Latin}]{2,}", def_token): continue1922 if def_token in particles or def_token in misc_stop_words: continue1923 if def_token in tokens: continue1924 def_token_labels[def_token].add(label)1925 for def_token, labels in def_token_labels.items():1926 cooc_words[def_token] += 0.01 * len(labels) * max_word_weight1927 is_wiki_word = "wikipedia" in cooc_words or "encyclopedia" in cooc_words1928 merged_cooc_words = sorted(cooc_words.items(), key=lambda x: x[1], reverse=True)1929 weighed_cooc_words = []1930 for cooc_word, cooc_score in merged_cooc_words:1931 cooc_prob = self.GetPhraseProb(phrase_prob_dbm, "en", cooc_word)1932 cooc_idf = math.log(cooc_prob) * -11933 cooc_score *= cooc_idf ** 21934 if tkrzw_dict.IsStopWord("en", cooc_word):1935 if tkrzw_dict.IsStopWord("en", norm_word):1936 cooc_score *= 0.31937 else:1938 cooc_score *= 0.11939 elif cooc_word in particles or cooc_word in misc_stop_words:1940 cooc_score *= 0.51941 elif is_wiki_word and cooc_word in wiki_stop_words:1942 cooc_score *= 0.21943 weighed_cooc_words.append((cooc_word, cooc_score))1944 sorted_cooc_words = sorted(weighed_cooc_words, key=lambda x: x[1], reverse=True)1945 final_cooc_words = []1946 for cooc_word, cooc_score in sorted_cooc_words:1947 if len(final_cooc_words) >= 16: break1948 hit = False1949 for label, word_dict in word_dicts:1950 if label in self.surfeit_labels: continue1951 if cooc_word in word_dict:1952 hit = True1953 break1954 if not hit: continue1955 final_cooc_words.append(cooc_word)1956 if final_cooc_words:1957 word_entry["cooccurrence"] = final_cooc_words1958 def CompensateInflections(self, entry, merged_dict, verb_words):1959 word = entry["word"]1960 root_verb = None1961 ing_value = entry.get("verb_present_participle")1962 if ing_value and ing_value.endswith("<ing"):1963 root_verb = ing_value[:-4]1964 for infl_name in inflection_names:1965 value = entry.get(infl_name)1966 if value and not regex.fullmatch(r"[-\p{Latin}0-9', ]+", value):1967 del entry[infl_name]1968 poses = set()1969 for item in entry["item"]:1970 poses.add(item["pos"])1971 if "verb" in poses and word.find(" ") >= 0 and not regex.search(r"[A-Z]", word):1972 tokens = self.tokenizer.Tokenize("en", word, False, False)1973 if len(tokens) > 1:1974 if not root_verb:1975 for token in tokens:1976 if token not in particles and token not in misc_stop_words and token in verb_words:1977 root_verb = token1978 break1979 if root_verb:1980 root_entry = merged_dict.get(root_verb)1981 if root_entry:1982 for infl_name in inflection_names:1983 if not infl_name.startswith("verb_") or entry.get(infl_name):1984 continue1985 root_infls = root_entry[0].get(infl_name)1986 if not root_infls:1987 continue1988 phrase_infls = []1989 for root_infl in regex.split(r"[,|]", root_infls):1990 root_infl = root_infl.strip()1991 if not root_infl: continue1992 root_infl_tokens = []1993 for token in tokens:1994 if root_infl and token == root_verb:1995 root_infl_tokens.append(root_infl)1996 root_infl = None1997 else:1998 root_infl_tokens.append(token)1999 phrase_infls.append(" ".join(root_infl_tokens))2000 if phrase_infls:2001 entry[infl_name] = ", ".join(phrase_infls)2002 def CompensateAlternatives(self, word_entry, merged_dict):2003 word = word_entry["word"]2004 alternatives = word_entry.get("alternative") or []2005 variants = self.GetSpellVariants(word)2006 wn_count = 02007 for item in word_entry["item"]:2008 if item["label"] != "wn": continue2009 wn_count += 12010 for section in item["text"].split("[-]"):2011 section = section.strip()2012 match = regex.search(r"\[synonym\]: (.*)", section)2013 if match:2014 for synonym in match.group(1).split(","):2015 synonym = synonym.strip()2016 dist = tkrzw.Utility.EditDistanceLev(word, synonym)2017 similar = False2018 if dist == 1 and word[:3] != synonym[:3]:2019 similar = True2020 elif dist == 2 and word[:5] == synonym[:5] and word[-2:] == synonym[-2:]:2021 similar = True2022 if similar and synonym not in variants:2023 variants.add(synonym)2024 for variant in variants:2025 if word[:2] != variant[:2]: continue2026 if variant in alternatives: continue2027 variant_entries = merged_dict.get(variant)2028 if not variant_entries: continue2029 for variant_entry in variant_entries:2030 if variant_entry["word"] != variant: continue2031 var_wn_count = 02032 var_wn_counts = collections.defaultdict(int)2033 for item in variant_entry["item"]:2034 if item["label"] != "wn": continue2035 var_wn_count += 12036 for section in item["text"].split("[-]"):2037 section = section.strip()2038 match = regex.search(r"\[synonym\]: (.*)", section)2039 if match:2040 for synonym in match.group(1).split(","):2041 synonym = synonym.strip()2042 if synonym:2043 var_wn_counts[synonym] += 12044 hits = var_wn_counts[word]2045 if (wn_count > 0 and var_wn_count == wn_count and hits == wn_count) or hits >= 4:2046 alternatives.append(variant)2047 if alternatives:2048 word_entry["alternative"] = alternatives2049 def GetSpellVariants(self, word):2050 variants = set()2051 suffix_pairs = [("se", "ze"), ("sing", "zing"), ("sed", "zed"), ("ser", "zer"),2052 ("sation", "zation"), ("ce", "se"),2053 ("our", "or"), ("og", "ogue"), ("re", "er"), ("l", "ll")]2054 for suffix1, suffix2 in suffix_pairs:2055 if word.endswith(suffix1):2056 variant = word[:-len(suffix1)] + suffix22057 variants.add(variant)2058 if word.endswith(suffix2):2059 variant = word[:-len(suffix2)] + suffix12060 variants.add(variant)2061 return variants2062 def GetEntryTranslations(self, merged_dict, word, is_capital, best_pos):2063 key = tkrzw_dict.NormalizeWord(word)2064 entry = merged_dict.get(key)2065 if not entry: return None2066 scored_trans = []2067 word_score = 1.02068 for word_entry in entry:2069 cmp_word = word_entry["word"]2070 if bool(regex.search(r"\p{Lu}", cmp_word)) != is_capital:2071 continue2072 item_score = 1.02073 for item in word_entry["item"]:2074 pos = item["pos"]2075 text = item["text"]2076 trans = self.ExtractTextLabelTrans(text)2077 if trans:2078 score = word_score * item_score2079 if pos == best_pos:2080 score *= 2.02081 for tran in trans:2082 scored_trans.append((tran, score))2083 score *= 0.92084 item_score *= 0.92085 trans = word_entry.get("translation")2086 if trans:2087 score = word_score * item_score2088 for tran in trans:2089 scored_trans.append((tran, score))2090 score *= 0.92091 word_score *= 0.52092 scored_trans = sorted(scored_trans, key=lambda x: x[1], reverse=True)2093 return [x[0] for x in scored_trans]2094 def PropagateTranslations(self, entry, merged_dict, tran_prob_dbm, aux_last_trans):2095 old_trans = entry.get("translation") or []2096 if len(old_trans) >= 8: return2097 word = entry["word"]2098 is_capital = bool(regex.search(r"\p{Lu}", word))2099 if len(word) <= 2: return2100 uniq_labels = set()2101 top_exprs = []2102 poses = set()2103 synonyms = []2104 for item in entry["item"]:2105 label = item["label"]2106 pos = item["pos"]2107 poses.add(pos)2108 if label in self.gross_labels or label in self.supplement_labels: continue2109 is_first = label not in uniq_labels2110 uniq_labels.add(label)2111 text = item["text"]2112 for field in text.split(" [-] "):2113 if not field.startswith("[synonym]: "): continue2114 field = regex.sub(r"^[^:]+: ", "", field)2115 field = regex.sub(r"\(.*?\) *", "", field)2116 for synonym in field.split(","):2117 synonym = synonym.strip()2118 if synonym:2119 synonyms.append((synonym, pos))2120 text = regex.sub(r" \[-+\] .*", "", text)2121 text = regex.sub(r"\(.*?\)", "", text)2122 text = regex.sub(r"\.$", "", text)2123 text = regex.sub(r"([-\p{Latin}\d]{5,})\.", r"\1;", text)2124 for expr in text.split(";"):2125 expr = expr.strip()2126 if pos == "verb":2127 expr = regex.sub(r"^to +([\p{Latin}])", r"\1", expr, flags=regex.IGNORECASE)2128 elif pos == "noun":2129 expr = regex.sub(r"^(a|an|the) +([\p{Latin}])", r"\2", expr, flags=regex.IGNORECASE)2130 if expr:2131 top_exprs.append((expr, pos, is_first))2132 top_words = []2133 for expr, pos, is_first in top_exprs:2134 manner_match = regex.search(r"^in +([-\p{Latin}].*?) +(manner|fashion|way)$",2135 expr, regex.IGNORECASE)2136 preps = ["of", "in", "at", "from", "by", "part of", "out of", "inside",2137 "relating to", "related to", "associated with",2138 "characterized by", "pertaining to", "derived from"]2139 prep_expr = None2140 for prep in preps:2141 if len(expr) > len(prep):2142 if expr[:len(prep)].lower() == prep:2143 expr_lead = expr[len(prep):]2144 joint_match = regex.match(r"^,?( +or)? +", expr_lead)2145 if joint_match:2146 expr = expr_lead[joint_match.end():]2147 prep_expr = expr2148 if manner_match:2149 expr = manner_match.group(1).strip()2150 expr = regex.sub(r"^(a|an|the) +", "", expr, flags=regex.IGNORECASE)2151 if expr:2152 top_words.append((expr, "adjective", "adverb", is_first))2153 elif prep_expr:2154 expr = regex.sub(r"^(a|an|the) +([\p{Latin}])", r"\2", prep_expr, flags=regex.IGNORECASE)2155 if expr:2156 new_pos = "adverb" if pos == "adverb" else "adjective"2157 top_words.append((expr, "noun", new_pos, is_first))2158 else:2159 expr = expr.strip()2160 if expr:2161 top_words.append((expr, pos, "", is_first))2162 etym_prefix = entry.get("etymology_prefix")2163 etym_core = entry.get("etymology_core")2164 etym_suffix = entry.get("etymology_suffix")2165 if ("noun" in poses and not etym_prefix and etym_core and2166 etym_suffix in ("ness", "cy", "ity")):2167 top_words.append((etym_core, "adjective", "noun", True))2168 if ("noun" in poses and not etym_prefix and etym_core and2169 etym_suffix in ("ment", "tion", "sion")):2170 top_words.append((etym_core, "verb", "noun", True))2171 if ("verb" in poses and not etym_prefix and etym_core and2172 etym_suffix in ("ise", "ize")):2173 top_words.append((etym_core, "adjective", "verb", True))2174 if ("adjective" in poses and not etym_prefix and etym_core2175 and etym_suffix in ("ic", "ical", "ish", "ly")):2176 top_words.append((etym_core, "noun", "adjective", True))2177 if ("adverb" in poses and not etym_prefix and etym_core and2178 etym_suffix == "ly"):2179 top_words.append((etym_core, "adjective", "adverb", True))2180 parents = entry.get("parent")2181 if parents:2182 for parent in parents:2183 if len(parent) < 5: continue2184 if ("noun" in poses and2185 (word.endswith("ness") or word.endswith("cy") or word.endswith("ity"))):2186 top_words.append((parent, "adjective", "noun", True))2187 if ("noun" in poses and2188 (word.endswith("ment") or word.endswith("tion") or word.endswith("sion"))):2189 top_words.append((parent, "verb", "noun", True))2190 if ("verb" in poses and2191 (word.endswith("ise") or word.endswith("tze"))):2192 top_words.append((parent, "adjective", "verb", True))2193 if ("adjective" in poses and2194 (word.endswith("ic") or word.endswith("ical") or word.endswith("ish"))):2195 top_words.append((parent, "noun", "adjective", True))2196 if ("adverb" in poses and2197 word.endswith("ly")):2198 top_words.append((parent, "adjective", "adverb", True))2199 ent_synonyms = entry.get("_synonym")2200 if ent_synonyms:2201 for synonym in ent_synonyms:2202 norm_synonym = tkrzw_dict.NormalizeWord(synonym)2203 syn_entries = merged_dict.get(norm_synonym)2204 if syn_entries:2205 syn_pos = ""2206 for syn_entry in syn_entries:2207 if syn_entry["word"] != synonym: continue2208 for syn_item in syn_entry["item"]:2209 if syn_item["pos"] in poses:2210 syn_pos = syn_item["pos"]2211 break2212 if syn_pos:2213 synonyms.append((synonym, syn_pos))2214 for synonym, pos in synonyms:2215 top_words.append((synonym, pos, "", False))2216 trans = []2217 tran_sources = set()2218 for expr, pos, conversion, trustable in top_words:2219 expr = regex.sub(r"^([-\p{Latin}]+), ([-\p{Latin}]+),? +or +([-\p{Latin}]+)$",2220 r"\1; \2; \3", expr)2221 expr = regex.sub(r"^([-\p{Latin}]+) +or +([-\p{Latin}]+)$", r"\1; \2", expr)2222 expr = regex.sub(r"^([-\p{Latin}]+), +([-\p{Latin}]+)$", r"\1; \2", expr)2223 for rel_word in expr.split(";"):2224 rel_word = rel_word.strip()2225 if len(rel_word) <= 2: continue2226 word_trans = self.GetEntryTranslations(merged_dict, rel_word, is_capital, pos)2227 if not word_trans: continue2228 new_pos = conversion or pos2229 if new_pos == "noun":2230 word_trans = [self.MakeTranNoun(x) for x in word_trans]2231 elif new_pos == "verb":2232 word_trans = [self.MakeTranVerb(x) for x in word_trans]2233 elif new_pos == "adjective":2234 word_trans = [self.MakeTranAdjective(x) for x in word_trans]2235 elif new_pos == "adverb":2236 word_trans = [self.MakeTranAdverb(x) for x in word_trans]2237 for rank, word_tran in enumerate(word_trans):2238 tran_source = (word_tran, rel_word)2239 if tran_source in tran_sources: continue2240 tran_sources.add(tran_source)2241 trans.append((word_tran, trustable, rel_word, rank))2242 prob_trans = {}2243 key = tkrzw_dict.NormalizeWord(word)2244 tsv = tran_prob_dbm.GetStr(key)2245 if tsv:2246 fields = tsv.split("\t")2247 for i in range(0, len(fields), 3):2248 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])2249 if regex.search("[っん]$", trg) and self.tokenizer.GetJaLastPos(trg)[1] == "動詞":2250 continue2251 norm_trg = tkrzw_dict.NormalizeWord(trg)2252 prob = float(prob)2253 if src != word:2254 prob *= 0.12255 prob_trans[norm_trg] = max(prob_trans.get(norm_trg) or 0.0, prob)2256 scored_trans = []2257 tran_counts = {}2258 for tran, trustable, rel_word, rank in trans:2259 tran_counts[tran] = (tran_counts.get(tran) or 0) + 12260 for tran, trustable, rel_word, rank in trans:2261 norm_tran = tkrzw_dict.NormalizeWord(tran)2262 max_weight = 02263 prob_hit = False2264 for prob_tran, prob in prob_trans.items():2265 prob **= 0.252266 dist = tkrzw.Utility.EditDistanceLev(norm_tran, prob_tran)2267 dist /= max(len(norm_tran), len(prob_tran))2268 weight = prob ** 0.5 + 2.0 - dist2269 if norm_tran == prob_tran:2270 weight *= 102271 prob_hit = True2272 elif len(prob_tran) >= 2 and norm_tran.startswith(prob_tran):2273 weight *= 52274 prob_hit = True2275 elif len(norm_tran) >= 2 and prob_tran.startswith(norm_tran):2276 weight *= 52277 prob_hit = True2278 elif len(prob_tran) >= 2 and norm_tran.find(prob_tran) >= 0:2279 weight *= 32280 prob_hit = True2281 elif len(norm_tran) >= 2 and prob_tran.find(norm_tran) >= 0:2282 weight *= 32283 prob_hit = True2284 elif dist < 0.3:2285 weight *= 22286 prob_hit = True2287 max_weight = max(max_weight, weight)2288 if not trustable and not prob_hit:2289 continue2290 tran_count = tran_counts[tran]2291 count_score = 1 + (tran_count * 0.2)2292 rank_score = 0.95 ** rank2293 score = max_weight * count_score * rank_score2294 scored_trans.append((tran, score, prob_hit))2295 scored_trans = sorted(scored_trans, key=lambda x: x[1], reverse=True)2296 rec_aux_trans = aux_last_trans.get(word)2297 if rec_aux_trans:2298 scored_aux_trans = []2299 for aux_tran in rec_aux_trans:2300 norm_trg = tkrzw_dict.NormalizeWord(aux_tran)2301 prob = prob_trans.get(norm_trg) or 0.02302 prob += 0.01 / (len(aux_tran) + 1)2303 scored_aux_trans.append((aux_tran, prob))2304 scored_aux_trans = sorted(scored_aux_trans, key=lambda x: x[1], reverse=True)2305 for aux_tran, score in scored_aux_trans:2306 scored_trans.append((aux_tran, 0, False))2307 final_trans = []2308 uniq_trans = set()2309 for tran in old_trans:2310 norm_tran = tkrzw_dict.NormalizeWord(tran)2311 uniq_trans.add(norm_tran)2312 final_trans.append(tran)2313 num_rank = 02314 for tran, score, prob_hit in scored_trans:2315 if len(final_trans) >= 8: break2316 norm_tran = tkrzw_dict.NormalizeWord(tran)2317 if norm_tran in uniq_trans: continue2318 num_rank += 12319 if not prob_hit:2320 if num_rank > 3: continue2321 if num_rank > 2 and len(final_trans) >= 3: continue2322 uniq_trans.add(norm_tran)2323 final_trans.append(tran)2324 if final_trans:2325 entry["translation"] = final_trans2326 def MakeTranNoun(self, tran):2327 pos = self.tokenizer.GetJaLastPos(tran)2328 stem = self.tokenizer.CutJaWordNounParticle(tran)2329 if tran.endswith("する"):2330 tran = tran[:-2]2331 elif tran.endswith("される"):2332 tran = tran[:-3]2333 elif tran.endswith("された"):2334 tran = tran[:-3]2335 elif tran.endswith("ような"):2336 tran = tran[:-3]2337 elif self.tokenizer.IsJaWordAdjvNoun(stem):2338 tran = stem2339 elif tran.endswith("い") and pos[1] == "形容詞":2340 tran = tran[:-1] + "さ"2341 elif pos[1] in "動詞" and regex.search(r"[うくすつぬふむゆる]$", tran):2342 tran = tran + "こと"2343 elif pos[1] in "形容詞" and regex.search(r"[きい]$", tran):2344 tran = tran + "こと"2345 elif pos[0] in ("た", "な") and pos[1] == "助動詞":2346 tran = tran + "こと"2347 return tran2348 def MakeTranVerb(self, tran):2349 pos = self.tokenizer.GetJaLastPos(tran)2350 if self.tokenizer.IsJaWordSahenNoun(tran):2351 tran = tran + "する"2352 elif tran.endswith("い") and pos[1] == "形容詞":2353 tran = tran[:-1] + "くする"2354 elif pos[1] == "名詞" and pos[2] == "形容動詞語幹":2355 tran = tran + "にする"2356 return tran2357 def MakeTranAdjective(self, tran):2358 pos = self.tokenizer.GetJaLastPos(tran)2359 stem = self.tokenizer.CutJaWordNounParticle(tran)2360 is_adjv = False2361 if tran.endswith("する"):2362 tran = tran[:-2]2363 elif tran.endswith("される"):2364 tran = tran[:-3]2365 elif tran.endswith("された"):2366 tran = tran[:-3]2367 elif tran.endswith("ような"):2368 tran = tran[:-3]2369 elif self.tokenizer.IsJaWordAdjvNoun(stem):2370 tran = stem2371 is_adjv = True2372 pos = self.tokenizer.GetJaLastPos(tran)2373 if self.tokenizer.IsJaWordAdjvNounOnly(tran):2374 tran += "な"2375 elif pos[1] == "名詞":2376 if tran.endswith("的"):2377 tran += "な"2378 else:2379 tran += "の"2380 return tran2381 def MakeTranAdverb(self, tran):2382 pos = self.tokenizer.GetJaLastPos(tran)2383 stem = self.tokenizer.CutJaWordNounParticle(tran)2384 if tran.endswith("する"):2385 tran = tran[:-2] + "して"2386 elif tran.endswith("される"):2387 tran = tran[:-3] + "されて"2388 elif tran.endswith("された"):2389 tran = tran[:-3] + "されて"2390 elif tran.endswith("ような"):2391 tran = tran[:-3] + "ように"2392 elif tran.endswith("らしい"):2393 tran = tran[:-3] + "らしく"2394 elif tran.endswith("とした"):2395 tran = tran[:-3] + "として"2396 elif tran.endswith("い") and pos[1] == "形容詞":2397 tran = tran[:-1] + "く"2398 elif tran.endswith("的な"):2399 tran = tran[:-1] + "に"2400 elif self.tokenizer.IsJaWordSahenNoun(stem):2401 tran = stem + "して"2402 elif self.tokenizer.IsJaWordAdjvNoun(stem):2403 tran = stem + "に"2404 elif stem != tran or pos[1] == "名詞":2405 tran = stem + "で"2406 elif pos[0] == "た" and pos[1] == "助動詞":2407 tran = tran[:-1] + "て"2408 elif pos[1] == "動詞":2409 tran = stem + "ように"2410 return tran2411 def SetPhraseTranslations(self, entry, merged_dict, aux_trans, aux_last_trans,2412 tran_prob_dbm, phrase_prob_dbm, noun_words, verb_words,2413 live_words, rev_live_words):2414 if not tran_prob_dbm or not phrase_prob_dbm:2415 return2416 word = entry["word"]2417 if not regex.fullmatch(r"[-\p{Latin}]+", word):2418 return2419 if len(word) < 2 or word in ("an", "the"):2420 return2421 is_noun = word in noun_words2422 is_verb = word in verb_words2423 word_prob = float(phrase_prob_dbm.GetStr(word) or 0.0)2424 if word_prob < 0.00001:2425 return2426 word_mod_prob = min(word_prob, 0.001)2427 norm_word = " ".join(self.tokenizer.Tokenize("en", word, True, True))2428 if word != norm_word:2429 return2430 phrases = []2431 for particle in particles:2432 phrase = word + " " + particle2433 phrase_prob = float(phrase_prob_dbm.GetStr(phrase) or 0.0)2434 ratio = phrase_prob / word_mod_prob2435 if is_verb and ratio >= 0.005:2436 for pron in ("me", "us", "you", "him", "her", "it", "them"):2437 pron_phrase = word + " " + pron + " " + particle2438 pron_phrase_prob = float(phrase_prob_dbm.GetStr(pron_phrase) or 0.0)2439 if pron_phrase_prob > 0.0:2440 phrase_prob += pron_phrase_prob * 2.02441 ratio = phrase_prob / word_mod_prob2442 phrases.append((phrase, True, ratio, ratio, phrase_prob))2443 if ratio >= 0.005:2444 for sub_particle in particles:2445 sub_phrase = phrase + " " + sub_particle2446 sub_phrase_prob = float(phrase_prob_dbm.GetStr(sub_phrase) or 0.0)2447 sub_ratio = max(sub_phrase_prob / phrase_prob, 0.01)2448 phrases.append((sub_phrase, True, max(sub_ratio, ratio),2449 ratio * (sub_ratio ** 0.005), sub_phrase_prob))2450 verb_prob = 0.02451 if is_verb:2452 for auxverb in ("not", "will", "shall", "can", "may", "must"):2453 auxverb_prob = float(phrase_prob_dbm.GetStr(auxverb + " " + word) or 0.0)2454 verb_prob += auxverb_prob2455 verb_prob *= 202456 for particle in particles:2457 phrase = particle + " " + word2458 phrase_prob = float(phrase_prob_dbm.GetStr(phrase) or 0.0)2459 if particle == "to":2460 phrase_prob -= verb_prob2461 ratio = phrase_prob / word_mod_prob2462 phrases.append((phrase, False, ratio, ratio, phrase_prob))2463 if is_noun:2464 for art in ("the", "a", "an"):2465 sub_phrase = particle + " " + art + " " + word2466 sub_phrase_prob = float(phrase_prob_dbm.GetStr(sub_phrase) or 0.0)2467 sub_ratio = sub_phrase_prob / word_mod_prob2468 phrases.append((sub_phrase, False, sub_ratio, sub_ratio, sub_phrase_prob))2469 it = live_words.MakeIterator()2470 it.Jump(word + " ")2471 while True:2472 rec = it.GetStr()2473 if not rec: break2474 phrase, phrase_prob = rec2475 if not phrase.startswith(word + " "): break2476 phrase_prob = float(phrase_prob)2477 ratio = phrase_prob / word_prob2478 if ratio >= 0.05:2479 phrases.append((phrase, True, ratio, ratio, phrase_prob))2480 it.Next()2481 it = rev_live_words.MakeIterator()2482 it.Jump(word + " ")2483 while True:2484 rec = it.GetStr()2485 if not rec: break2486 phrase, phrase_prob = rec2487 if not phrase.startswith(word + " "): break2488 phrase_prob = float(phrase_prob)2489 ratio = phrase_prob / word_prob2490 if ratio >= 0.05:2491 phrase = " ".join(reversed(phrase.split(" ")))2492 phrases.append((phrase, True, ratio, ratio, phrase_prob))2493 it.Next()2494 if not phrases:2495 return2496 orig_trans = {}2497 tsv = tran_prob_dbm.GetStr(word)2498 if tsv:2499 fields = tsv.split("\t")2500 for i in range(0, len(fields), 3):2501 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])2502 trg = regex.sub(r"[～〜]", "", trg)2503 trg, trg_prefix, trg_suffix = self.tokenizer.StripJaParticles(trg)2504 if src == word and prob >= 0.06:2505 orig_trans[trg] = prob2506 aux_orig_trans = (aux_trans.get(word) or []) + (aux_last_trans.get(word) or [])2507 if aux_orig_trans:2508 for trg in set(aux_orig_trans):2509 trg = regex.sub(r"[～〜]", "", trg)2510 trg, trg_prefix, trg_suffix = self.tokenizer.StripJaParticles(trg)2511 orig_trans[trg] = float(orig_trans.get(trg) or 0) + 0.12512 ent_orig_trans = entry.get("translation")2513 if ent_orig_trans:2514 base_score = 0.12515 for ent_orig_tran in ent_orig_trans:2516 orig_trans[ent_orig_tran] = float(orig_trans.get(ent_orig_tran) or 0) + base_score2517 base_score *= 0.92518 final_phrases = []2519 uniq_phrases = set()2520 for phrase, is_suffix, mod_prob, phrase_score, raw_prob in phrases:2521 if phrase in uniq_phrases: continue2522 uniq_phrases.add(phrase)2523 phrase_trans = {}2524 phrase_prefixes = {}2525 pos_match = is_verb if is_suffix else is_noun2526 if mod_prob >= 0.02:2527 if pos_match:2528 tsv = tran_prob_dbm.GetStr(phrase)2529 if tsv:2530 fields = tsv.split("\t")2531 for i in range(0, len(fields), 3):2532 src, trg, prob = fields[i], fields[i + 1], float(fields[i + 2])2533 if src != phrase:2534 continue2535 if regex.search("[っん]$", trg) and self.tokenizer.GetJaLastPos(trg)[1] == "動詞":2536 continue2537 if (is_verb and regex.search("[いきしちにひみり]$", trg) and2538 self.tokenizer.GetJaLastPos(trg)[1] == "動詞"):2539 continue2540 trg = regex.sub(r"[～〜]", "", trg)2541 trg, trg_prefix, trg_suffix = self.tokenizer.StripJaParticles(trg)2542 if not trg or regex.fullmatch(r"[\p{Katakana}ー]+", trg):2543 continue2544 pos = self.tokenizer.GetJaLastPos(trg)2545 if (is_noun and is_suffix and pos[1] == "名詞" and2546 not self.tokenizer.IsJaWordSahenNoun(trg)):2547 continue2548 if is_noun and is_suffix and trg in ("ある", "いる", "です", "ます"):2549 continue2550 orig_prob = orig_trans.get(trg) or 0.02551 if is_verb:2552 if self.tokenizer.IsJaWordSahenNoun(trg):2553 orig_prob = max(orig_prob, orig_trans.get(trg + "する") or 0.0)2554 for ext_suffix in ("する", "した", "して", "される", "された", "されて"):2555 orig_prob = max(orig_prob, orig_trans.get(trg[:len(ext_suffix)]) or 0.0)2556 if (is_suffix and is_verb and not trg_prefix and trg_suffix and2557 (pos[1] == "動詞" or self.tokenizer.IsJaWordSahenNoun(trg))):2558 trg_prefix = trg_suffix2559 trg_suffix = ""2560 elif is_suffix and is_noun and not trg_prefix:2561 if trg_suffix == "のため":2562 trg_suffix = "ための"2563 trg_prefix = trg_suffix2564 trg_suffix = ""2565 elif not trg_suffix and trg_prefix in ("ための", "のため"):2566 if trg.endswith("する"):2567 trg += "ための"2568 else:2569 trg += "のため"2570 trg_prefix = ""2571 elif trg_suffix:2572 trg += trg_suffix2573 sum_prob = orig_prob + prob2574 if sum_prob >= 0.1:2575 if is_verb and pos[1] == "動詞":2576 sum_prob += 0.12577 phrase_trans[trg] = float(phrase_trans.get(trg) or 0.0) + sum_prob2578 if trg_prefix and not trg_suffix:2579 part_key = trg + ":" + trg_prefix2580 phrase_prefixes[part_key] = float(phrase_trans.get(part_key) or 0.0) + sum_prob2581 for aux_phrase_trans in (aux_trans.get(phrase), aux_last_trans.get(phrase)):2582 if aux_phrase_trans:2583 for trg in aux_phrase_trans:2584 trg = regex.sub(r"[～〜]", "", trg)2585 trg, trg_prefix, trg_suffix = self.tokenizer.StripJaParticles(trg)2586 if is_noun and is_suffix and trg in ("ある", "いる", "です", "ます"):2587 continue2588 phrase_trans[trg] = float(phrase_trans.get(trg) or 0.0) + 0.12589 if mod_prob >= 0.001:2590 phrase_entries = merged_dict.get(phrase)2591 if phrase_entries:2592 for phrase_entry in phrase_entries:2593 if phrase_entry["word"] != phrase: continue2594 ent_phrase_trans = phrase_entry.get("translation")2595 if ent_phrase_trans:2596 base_score = 0.152597 for trg in ent_phrase_trans:2598 trg, trg_prefix, trg_suffix = self.tokenizer.StripJaParticles(trg)2599 phrase_trans[trg] = float(phrase_trans.get(trg) or 0.0) + base_score2600 if trg_prefix and not trg_suffix:2601 part_key = trg + ":" + trg_prefix2602 phrase_prefixes[part_key] = float(phrase_trans.get(part_key) or 0.0) + base_score2603 base_score *= 0.92604 if not phrase_trans:2605 continue2606 for tran in list(phrase_trans.keys()):2607 if not regex.search(r"[\p{Han}\p{Katakana}]", tran):2608 continue2609 for cmp_tran, cmp_score in list(phrase_trans.items()):2610 if cmp_tran not in phrase_trans: continue2611 if cmp_tran.startswith(tran):2612 suffix = cmp_tran[len(tran):]2613 if suffix in ("する", "される", "をする", "に", "な", "の"):2614 phrase_trans[cmp_tran] = cmp_score + float(phrase_trans.get(tran) or 0)2615 if tran in phrase_trans:2616 del phrase_trans[tran]2617 mod_trans = {}2618 for tran, score in phrase_trans.items():2619 prefix_check = tran + ":"2620 best_prefix = ""2621 best_prefix_score = 0.02622 for prefix, score in phrase_prefixes.items():2623 if not prefix.startswith(prefix_check): continue2624 if score >= best_prefix_score:2625 best_prefix = prefix[len(prefix_check):]2626 best_prefix_score = score2627 if regex.search(r"^[\p{Katakana}ー]", tran):2628 score *= 0.52629 pos = self.tokenizer.GetJaLastPos(tran)2630 if is_suffix and is_verb:2631 if pos[1] == "動詞" and regex.search("[うくすつぬふむゆる]$", tran):2632 score *= 1.52633 if pos[1] == "名詞" and not self.tokenizer.IsJaWordSahenNoun(tran):2634 score *= 0.52635 if not is_suffix and pos[1] == "名詞" and not best_prefix:2636 if self.tokenizer.IsJaWordSahenNoun(tran) or self.tokenizer.IsJaWordAdjvNoun(tran):2637 score *= 0.72638 else:2639 score *= 0.52640 if len(tran) <= 1:2641 score *= 0.52642 if is_verb:2643 orig_tran = tran2644 pos = self.tokenizer.GetJaLastPos(tran)2645 if self.tokenizer.IsJaWordSahenNoun(tran) and best_prefix != "の":2646 tran = tran + "する"2647 if best_prefix and best_prefix not in ("を", "が", "は"):2648 tran = "({}){}".format(best_prefix, tran)2649 mod_trans[tran] = float(mod_trans.get(tran) or 0.0) + score2650 scored_trans = sorted(mod_trans.items(), key=lambda x: x[1], reverse=True)[:4]2651 if scored_trans:2652 final_phrases.append((phrase, phrase_score, raw_prob, [x[0] for x in scored_trans]))2653 if final_phrases:2654 final_phrases = sorted(final_phrases, key=lambda x: x[1], reverse=True)2655 map_phrases = []2656 for phrase, score, raw_prob, trans in final_phrases:2657 prob_expr = "{:.6f}".format(raw_prob / word_prob).replace("0.", ".")2658 map_phrase = {"w": phrase, "p": prob_expr, "x": trans}2659 if phrase in merged_dict:2660 map_phrase["i"] = "1"2661 map_phrases.append(map_phrase)2662 entry["phrase"] = map_phrases2663 def FilterParents(self, word_entry, merged_dict):2664 word = word_entry["word"]2665 parents = word_entry.get("parent")2666 if not parents or len(parents) < 2: return2667 ancestors = parents2668 while True:2669 grand_ancestors = []2670 for ancestor in ancestors:2671 ancestor_entries = merged_dict.get(ancestor)2672 if ancestor_entries:2673 for ancestor_entry in ancestor_entries:2674 if ancestor_entry["word"] != ancestor: continue2675 for grand_ancestor in ancestor_entry.get("parent") or []:2676 if grand_ancestor in ancestors and grand_ancestor not in grand_ancestors:2677 grand_ancestors.append(grand_ancestor)2678 if not grand_ancestors or len(grand_ancestors) == len(ancestors):2679 break2680 ancestors = grand_ancestors2681 scored_parents = []2682 for parent in parents:2683 parent_prob = 02684 parent_entries = merged_dict.get(parent)2685 if parent_entries:2686 for parent_entry in parent_entries:2687 if parent_entry["word"] != parent: continue2688 parent_prob = float(parent_entry.get("probability")) or 02689 score = parent_prob + 0.0000012690 if word.startswith(parent):2691 score *= 22692 if parent in ancestors:2693 score += 12694 else:2695 is_dup = False2696 for suffix in ("ing", "ed", "er", "or", "ism", "ist", "est"):2697 for ancestor in ancestors:2698 candidate = ancestor + suffix2699 if (parent[:3] == candidate[:3] and2700 tkrzw.Utility.EditDistanceLev(parent, candidate) < 2):2701 is_dup = True2702 if is_dup:2703 continue2704 scored_parents.append((parent, score))2705 scored_parents = sorted(scored_parents, key=lambda x: x[1], reverse=True)2706 word_entry["parent"] = [x[0] for x in scored_parents]2707 def AbsorbInflections(self, word_entry, merged_dict):2708 word = word_entry["word"]2709 infls = []2710 for infl_name in inflection_names:2711 infl_value = word_entry.get(infl_name)2712 if infl_value:2713 for infl in infl_value.split(","):2714 infl = infl.strip()2715 if infl and infl != word and infl not in infls:2716 infls.append(infl)2717 phrases = []2718 for infl in infls:2719 infl_entries = merged_dict.get(infl)2720 if not infl_entries: continue2721 for infl_entry in infl_entries:2722 if infl_entry["word"] != infl: continue2723 is_core = False2724 good_labels = set()2725 num_good_items = 02726 for infl_item in infl_entry["item"]:2727 label = infl_item["label"]2728 text = infl_item["text"]2729 if label in self.supplement_labels: continue2730 if regex.search(r"^\[\w+]:", text): continue2731 good_labels.add(label)2732 if label in self.core_labels:2733 is_core = True2734 num_good_items += 12735 alive = True2736 if len(good_labels) < 2 and not is_core and num_good_items < 3:2737 infl_entry["deleted"] = True2738 alive = False2739 infl_trans = infl_entry.get("translation")2740 if infl_trans:2741 phrase = {"w": infl, "x": infl_trans[:4]}2742 if alive:2743 phrase["i"] = "1"2744 phrases.append(phrase)2745 if phrases:2746 old_phrases = word_entry.get("phrase")2747 if old_phrases:2748 phrases = phrases + old_phrases2749 word_entry["phrase"] = phrases2750def main():2751 args = sys.argv[1:]2752 output_path = tkrzw_dict.GetCommandFlag(args, "--output", 1) or "union-body.tkh"2753 core_labels = set((tkrzw_dict.GetCommandFlag(args, "--core", 1) or "xa,wn").split(","))2754 full_def_labels = set((tkrzw_dict.GetCommandFlag(2755 args, "--full_def", 1) or "ox,wn,we").split(","))2756 gross_labels = set((tkrzw_dict.GetCommandFlag(args, "--gross", 1) or "wj").split(","))2757 top_labels = set((tkrzw_dict.GetCommandFlag(args, "--top", 1) or "we,lx,xa").split(","))2758 slim_labels = set((tkrzw_dict.GetCommandFlag(args, "--slim", 1) or "ox,we,wj").split(","))2759 surfeit_labels = set((tkrzw_dict.GetCommandFlag(args, "--surfeit", 1) or "we").split(","))2760 tran_list_labels = set((tkrzw_dict.GetCommandFlag(2761 args, "--tran_list", 1) or "xa,wn,we").split(","))2762 supplement_labels = set((tkrzw_dict.GetCommandFlag(args, "--supplement", 1) or "xs").split(","))2763 phrase_prob_path = tkrzw_dict.GetCommandFlag(args, "--phrase_prob", 1) or ""2764 tran_prob_path = tkrzw_dict.GetCommandFlag(args, "--tran_prob", 1) or ""2765 tran_aux_paths = (tkrzw_dict.GetCommandFlag(args, "--tran_aux", 1) or "").split(",")2766 tran_aux_last_paths = (tkrzw_dict.GetCommandFlag(args, "--tran_aux_last", 1) or "").split(",")2767 rev_prob_path = tkrzw_dict.GetCommandFlag(args, "--rev_prob", 1) or ""2768 cooc_prob_path = tkrzw_dict.GetCommandFlag(args, "--cooc_prob", 1) or ""2769 aoa_paths = (tkrzw_dict.GetCommandFlag(args, "--aoa", 1) or "").split(",")2770 keyword_path = tkrzw_dict.GetCommandFlag(args, "--keyword", 1) or ""2771 min_prob_exprs = tkrzw_dict.GetCommandFlag(args, "--min_prob", 1) or ""2772 min_prob_map = {}2773 for min_prob_expr in min_prob_exprs.split(","):2774 columns = min_prob_expr.split(":")2775 if len(columns) == 2:2776 min_prob_map[columns[0]] = float(columns[1])2777 if tkrzw_dict.GetCommandFlag(args, "--quiet", 0):2778 logger.setLevel(logging.ERROR)2779 unused_flag = tkrzw_dict.GetUnusedFlag(args)2780 if unused_flag:2781 raise RuntimeError("Unknow flag: " + unused_flag)2782 inputs = tkrzw_dict.GetArguments(args)2783 if not inputs:2784 raise RuntimeError("inputs are required")2785 input_confs = []2786 for input in inputs:2787 input_conf = input.split(":", 1)2788 if len(input_conf) != 2:2789 raise RuntimeError("invalid input: " + input)2790 input_confs.append(input_conf)2791 BuildUnionDBBatch(input_confs, output_path, core_labels, full_def_labels, gross_labels,2792 surfeit_labels, top_labels, slim_labels, tran_list_labels, supplement_labels,2793 phrase_prob_path, tran_prob_path, tran_aux_paths, tran_aux_last_paths,2794 rev_prob_path, cooc_prob_path, aoa_paths, keyword_path,2795 min_prob_map).Run()2796if __name__=="__main__":...

Full Screen

Full Screen

shorte_engine.py

Source:shorte_engine.py Github

copy

Full Screen

...377 def add_wikiword(self, word):378 if(self.m_wiki_links.has_key(word)):379 FATAL("Wikiword %s already exists" % word)380 self.m_wiki_links[word.wikiword] = word381 def is_wiki_word(self, phrase):382 '''Returns the target link if the phrase is a wikiword383 or None if it does not exist'''384 link = None385 if(self.m_wiki_links.has_key(phrase)):386 link = self.m_wiki_links[phrase]387 return link388 def inkscape_to_png(self, name):389 '''This method is called to convert an inkscape390 SVG to PNG format for embedding in a document'''391 input = os.path.abspath(name)392 parts = os.path.splitext(input)393 basename = parts[0]394 #print "input = %s" % input395 #print "basename = %s" % basename...

Full Screen

Full Screen

template_sql.py

Source:template_sql.py Github

copy

Full Screen

...294 if(tmp == word):295 exclude_word = True296 break297 if(not exclude_word):298 link = self.m_engine.is_wiki_word(word)299 if(link != None):300 tmp = "<a href='%s#%s'>%s</a>" % (self.get_output_path(link), word, word)301 #print "WIKIWORD: %s" % tmp302 output += tmp303 else:304 #print "HERE I AM: %s" % word305 output += word306 return output307 308 def format_text(self, data, allow_wikify=True, exclude_wikify=[], expand_equals_block=False):309 if(data == None):310 return311 if(len(data) != 0):312 data = re.sub("'", "&apos;", data)...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Nose automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful