How to use update_mapping method in tempest

Best Python code snippet using tempest_python

build_index.py

Source:build_index.py Github

copy

Full Screen

1#!/usr/bin/env python2# -*- coding: utf-8 -*-3'''4TODO:5 score-tuning: 1. Symbol6 2. any ID7 3. name8 4. summary9 5. other text (uniprot name, interpro desc, go term)10 case to consider:11 q=hypoxia inducible factor12 if matching from the beginning, rank higher?13 appear first # name: "hypoxia-inducible factor 1, alpha subunit (basic helix-loop-helix transcription factor)"14 appear latter # name: "similar to egl nine homolog 1 (hypoxia-inducible factor prolyl hydroxylase 2) (hif-prolyl hydroxylase 2) (hif-ph2) (hph-2) (sm-20)"15 auto-correction: 1. multiple terms use default AND first, if no hit, do the OR automatically (with a note on UI)16'''17from functools import partial18from django.conf import settings19from django.db.models.signals import post_save, post_delete20from pyes.exceptions import (NotFoundException, IndexMissingException,21 ElasticSearchException, TypeMissingException)22from biogps.utils import ask23from biogps.utils.models import queryset_iterator24from es_lib import get_es_conn25import logging26log = logging.getLogger('pyes')27if settings.DEBUG:28 log.setLevel(logging.DEBUG)29 if len(log.handlers) == 0:30 log_handler = logging.StreamHandler()31 log.addHandler(log_handler)32class BiogpsESIndexerBase(object):33 ES_HOST = settings.ES_HOST34 ES_INDEX_NAME = settings.ES_INDEXES['default']35 step = 10036 def __init__(self):37 self.conn = get_es_conn(self.ES_HOST, default_idx=[self.ES_INDEX_NAME])38 def check(self):39 '''print out ES server info for verification.'''40 print "Servers:", self.conn.servers41 print "Default indices:", self.conn.default_indices42 print "ES_INDEX_TYPE:", self.ES_INDEX_TYPE43 def create_index(self):44 try:45 print self.conn.open_index(self.ES_INDEX_NAME)46 #except NotFoundException:47 except IndexMissingException:48 print self.conn.create_index(self.ES_INDEX_NAME)49 def delete_index_type(self, index_type):50 '''Delete all indexes for a given index_type.'''51 index_name = self.ES_INDEX_NAME52 #Check if index_type exists53 try:54 mapping = self.conn.get_mapping(index_type, index_name)55 except TypeMissingException:56 print 'Error: index type "%s" does not exist in index "%s".' % (index_type, index_name)57 return58 path = '/%s/%s' % (index_name, index_type)59 if ask('Confirm to delete all data under "%s":' % path) == 'Y':60 return self.conn.delete_mapping(index_name, index_type)61 def verify_mapping(self, update_mapping=False):62 '''Verify if index and mapping exist, update mapping if mapping does not exist,63 or "update_mapping=True" explicitly64 '''65 conn = self.conn66 index_name = self.ES_INDEX_NAME67 index_type = self.ES_INDEX_TYPE68 #Test if index exists69 try:70 print "Opening index...", conn.open_index(index_name)71 except NotFoundException:72 print 'Error: index "%s" does not exist. Create it first.' % index_name73 return -174 try:75 cur_mapping = conn.get_mapping(index_type, index_name)76 empty_mapping = False77 except ElasticSearchException:78 #if no existing mapping available for index_type79 #force update_mapping to True80 empty_mapping = True81 update_mapping = True82# empty_mapping = not cur_mapping[index_name].get(index_type, {})83# if empty_mapping:84# #if no existing mapping available for index_type85# #force update_mapping to True86# update_mapping = True87 if update_mapping:88 print "Updating mapping...",89 if not empty_mapping:90 print "\n\tRemoving existing mapping...",91 print conn.delete_mapping(index_name, index_type)92 _mapping = self.get_field_mapping()93 print conn.put_mapping(index_type,94 _mapping,95 [index_name])96 def index(self, doc, index_type, id=None):97 '''add a doc to the index. If id is not None, the existing doc will be98 updated.99 '''100 return self.conn.index(doc, self.ES_INDEX_NAME, index_type, id=id)101 def delete_index(self, index_type, id):102 '''delete a doc from the index based on passed id.'''103 return self.conn.delete(self.ES_INDEX_NAME, index_type, id)104 def optimize(self):105 return self.conn.optimize(self.ES_INDEX_NAME, wait_for_merge=True)106class BiogpsModelESIndexer(BiogpsESIndexerBase):107 '''The base class for indexing objects from BioGPSModel derived models,108 e.g., BioGPSPlugin, BiogpsGenereportLayout, etc.109 '''110 _model = None # need to specify in each subclass111 ES_INDEX_TYPE = None # need to specify in each subclass112 step = 100 # how many objects to retrieve in one queryset query113 def _get_field_mapping(self, extra_attrs={}):114 #field mapping templates115 id_type = {'store': "yes",116 'index': 'not_analyzed',117 'type': 'string',118 'term_vector': 'with_positions_offsets'}119 text_type = {'store': "no",120 'index': 'analyzed',121 'type': 'string',122 'term_vector': 'with_positions_offsets'}123 date_type = {'store': "no",124 'index': 'not_analyzed',125 'type': 'date',126 'format': 'YYYY-MM-dd HH:mm:ss'}127 integer_type = {'type': 'integer'}128 float_type = {'type': 'float'}129 object_type = {'type': 'object'}130 boolean_type = {'type': 'boolean'}131 store_only = {'store': "yes",132 'index': 'no',133 'type': 'string'}134 disabled_object = {'type': 'object',135 'enabled': False}136 disabled_string = {'type': 'string',137 'index': 'no'}138 disabled_double = {'type': 'double',139 'index': 'no'}140 disabled_integer = {'type': 'integer',141 'index': 'no'}142 td = {'id_type': id_type,143 'text_type': text_type,144 'date_type': date_type,145 'integer_type': integer_type,146 'float_type': float_type,147 'boolean_type': boolean_type,148 'object_type': object_type,149 'store_only': store_only,150 'disabled_object': disabled_object,151 'disabled_string': disabled_string,152 'disabled_double': disabled_double,153 'disabled_integer': disabled_integer}154 properties = {'in': id_type,155 'id': id_type,156 'created': date_type,157 'lastmodified': date_type,158 'role_permission': id_type,159 'tags': id_type}160 for t_id in td.keys():161 for attr in extra_attrs.pop(t_id, []):162 properties[attr] = td[t_id]163 properties.update(extra_attrs)164 for f in properties:165 properties[f] = properties[f].copy()166 #some special settings167 #for tag field168 properties['tags']['index_name'] = 'tag'169 #for name field170 properties['name'] = {171 "type" : "multi_field",172 "store": "yes",173 "fields" : {174 "name" : {175 'index': 'analyzed',176 'type': 'string',177 'boost': 2.0,178 'term_vector': 'with_positions_offsets'179 },180 "for_sort" : {181 "type" : "string",182 "index" : "not_analyzed"183 }184 }185 }186 #for owner field187 properties['owner'] = {188 "store": "yes",189 "type": "object",190 "path": 'just_name', #! This is important to make query "username:cwudemo" work, instead of using "owner.username:cwudemo"191 "properties" : {192 "username" : {193 "type" : "string",194 "index_name": "username",195 "index": "not_analyzed",196 },197 "name" : {198 "type" : "string",199 "index_name": "owner",200 "index": "analyzed",201 },202 "url" : {203 "type" : "string",204 "index": "no",205 }206 }207 }208 #for rating_data field209 properties['rating_data'] = {210 "store": "yes",211 "type": "object",212 "properties": {213 "avg_stars": {"type": "short"},214 "total": {"type": "short"},215 "avg": {"type": "short"},216 }217 }218 mapping = {'properties': properties}219 # enable _source compression220 mapping["_source"] = {"enabled" : True,221 "compress": True,222 "compression_threshold": "1kb"}223# #store "_all" for highlighting.224# mapping["_all"] = {"store": "yes",225# "type": "string",226# "term_vector": "with_positions_offsets"}227 return mapping228 def get_field_mapping(self):229 raise NotImplementedError230 def build_index(self, update_mapping=False, bulk=True, verbose=False):231 conn = self.conn232 index_name = self.ES_INDEX_NAME233 index_type = self.ES_INDEX_TYPE234 self.verify_mapping(update_mapping=update_mapping)235 print "Building index..."236 cnt = 0237 for p in queryset_iterator(self._model, batch_size=self.step):238 doc = p.object_cvt(mode='es')239 conn.index(doc, index_name, index_type, doc['id'], bulk=bulk)240 cnt += 1241 if verbose:242 print cnt, ':', doc['id']243 print conn.flush()244 print conn.refresh()245 print 'Done! - {} docs indexed.'.format(cnt)246 def index_object(self, object_id):247 obj = self._model.objects.get(id=object_id)248 doc = obj.object_cvt(mode='es')249 print self.index(doc, self.ES_INDEX_TYPE, id=doc['id'])250class BiogpsPluginESIndexer(BiogpsModelESIndexer):251 '''A class for indexing all BiogpsPlugin objects.'''252 def __init__(self):253 from biogps.plugin.models import BiogpsPlugin254 self._model = BiogpsPlugin255 self.ES_INDEX_TYPE = self._model.short_name256 super(BiogpsModelESIndexer, self).__init__()257 def get_field_mapping(self):258 m_usage_data = {259 "store": "yes",260 "type": "object",261 "properties": {262 "users": {"type": "short"},263 "layouts": {"type": "short"},264 }265 }266 m = self._get_field_mapping(extra_attrs={'id_type': ['type', 'species'],267 'text_type': ['name', 'description', 'short_description', 'url'],268 "float_type": ['popularity'],269 'disabled_object': ['options'],270 'disabled_string': ['shortUrl', 'permission_style'],271 'usage_data': m_usage_data272 })273 return m274class BiogpsLayoutESIndexer(BiogpsModelESIndexer):275 '''A class for indexing all BiogpsGenereportLayout objects.'''276 def __init__(self):277 from biogps.layout.models import BiogpsGenereportLayout278 self._model = BiogpsGenereportLayout279 self.ES_INDEX_TYPE = self._model.short_name280 super(BiogpsModelESIndexer, self).__init__()281 def get_field_mapping(self):282 m = self._get_field_mapping(extra_attrs={'text_type': ['name', 'description'],283 'disabled_string': ['permission_style'],284 })285 #some special settings286# m['name']['boost'] = 2.0287 return m288class BiogpsGenelistESIndexer(BiogpsModelESIndexer):289 '''A class for indexing all BiogpsGeneList objects.'''290 def __init__(self):291 from biogps.genelist.models import BiogpsGeneList292 self._model = BiogpsGeneList293 self.ES_INDEX_TYPE = self._model.short_name294 super(BiogpsModelESIndexer, self).__init__()295 def get_field_mapping(self):296 m = self._get_field_mapping(extra_attrs={'text_type': ['name', 'description'],297 'disabled_string': ['permission_style'],298 })299 #some special settings300# m['name']['boost'] = 2.0301 return m302def _rebuild_x(delete_old=False, update_mapping=False, indexer=None):303 '''A convenient function for re-building indexes.304 '''305 es_indexer = indexer()306 if delete_old:307 es_indexer.delete_index_type(es_indexer.ES_INDEX_TYPE)308 update_mapping = True # if delete_old is True, update_mapping should be True anyway309 es_indexer.build_index(update_mapping=update_mapping, bulk=True)310rebuild_plugin = partial(_rebuild_x, indexer=BiogpsPluginESIndexer)311rebuild_plugin.__doc__ = 'A convenient function for re-building all plugins.'312rebuild_layout = partial(_rebuild_x, indexer=BiogpsLayoutESIndexer)313rebuild_layout.__doc__ = 'A convenient function for re-building all layouts.'314rebuild_genelist = partial(_rebuild_x, indexer=BiogpsGenelistESIndexer)315rebuild_genelist.__doc__ = 'A convenient function for re-building all genelists.'316def rebuild_all(delete_old=False):317 '''A convenient function for re-building all biogpsmodel objects,318 not including genes and datasets.319 '''320 rebuild_plugin(delete_old)321 rebuild_layout(delete_old)322 rebuild_genelist(delete_old)323def on_the_fly_es_update_handler(sender, **kwargs):324 '''A post_save signal handler to update ES indexes whenever a BiogpsModel object325 is created/updated.326 To connect to a signal:327 post_save.connect(on_the_fly_es_update_handler, sender=BioGPSModel,328 dispatch_uid="some_unique_id")329 '''330 if not getattr(settings, "SUSPEND_ES_UPDATE", None):331 object = kwargs['instance']332 es_indexer = BiogpsModelESIndexer()333 doc = object.object_cvt(mode='es')334 res = es_indexer.index(doc, object.short_name, object.id)335 return True336def on_the_fly_es_delete_handler(sender, **kwargs):337 '''A post_delete signal handler to delete ES indexes whenever a BiogpsModel object338 is deleted.339 To connect to a signal:340 post_delete.connect(on_the_fly_es_delete_handler, sender=BioGPSModel,341 dispatch_uid="some_unique_id")342 '''343 if not getattr(settings, "SUSPEND_ES_UPDATE", None):344 object = kwargs['instance']345 es_indexer = BiogpsModelESIndexer()346 res = es_indexer.delete_index(object.short_name, object.id)347 return True348def set_on_the_fly_indexing(biogpsmodel):349 '''set input biogpsmodel for on the fly indexing:350 * add to index when a new object is created351 * update index when an object is updated352 * delete from index when an object is deleted353 '''354 post_save.connect(on_the_fly_es_update_handler, sender=biogpsmodel,355 dispatch_uid=biogpsmodel.__name__ + 'update_indexer')356 post_delete.connect(on_the_fly_es_delete_handler, sender=biogpsmodel,...

Full Screen

Full Screen

feature_base.py

Source:feature_base.py Github

copy

Full Screen

...28 A set of features that generates a pattern for the features it contains.29 This set will act like a mapping in that we map names to patterns.30 """31 mapping = {}32 def update_mapping(self):33 u"""34 Called every time we care about the mapping of names to features.35 """36 self.mapping = dict([(f.name, f) for f in iter(self)])37 @property38 def PATTERN(self):39 u"""40 Uses the mapping of names to features to return a PATTERN suitable41 for using the lib2to3 patcomp.42 """43 self.update_mapping()44 return u" |\n".join([pattern_unformatted % (f.name, f._pattern) for f in iter(self)])45 def __getitem__(self, key):46 u"""47 Implement a simple mapping to get patterns from names.48 """...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run tempest automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful