How to use section_logger method in molecule

Best Python code snippet using molecule_python

visualgenome.py

Source:visualgenome.py Github

copy

Full Screen

...51 "|".join(r['synsets'])52 ]53 return extractor54def extract_relationship_dataframe(vg, limit=10, report=2e5):55 section = section_logger(1)56 data = []57 counter = 058 for img in vg['relationships']:59 if counter > limit:60 break61 else:62 counter += 163 if counter % report == 0:64 section("Loaded relationships in {} images".format(counter))65 rows = map(extract_relationship_data(img['image_id']), img['relationships'])66 data.extend(list(rows))67 rdf = pd.DataFrame(data, columns=['img', 'object', 'subject', 'predicate', 'synsets'])68 return rdf69def extract_object_data(img_id, img_w, img_h):70 def extractor(obj):71 return [72 obj['object_id'],73 "|".join(obj['synsets']),74 "|".join(obj['names']),75 img_id,76 obj['x']/img_w,77 obj['y']/img_h,78 obj['h']/img_h,79 obj['w']/img_w80 ]81 return extractor82def extract_object_dataframe(vg, limit=10, report=2e5):83 section = section_logger(1)84 data = []85 counter = 086 for img in vg['objects']:87 if counter > limit:88 break89 else:90 counter += 191 if counter % report == 0:92 section("Loaded objects in {} images".format(counter))93 current_image = vg['imageIdMap'][img['image_id']]94 assert current_image['image_id'] == img['image_id'], 'Position in the list {} does not correspond to ID {}'.format(img['image_id'], current_image['image_id'])95 rows = map(extract_object_data(img['image_id'], current_image['width'], current_image['height']), img['objects'])96 data.extend(rows)97 odf = pd.DataFrame(data, columns=['object_id', 'synsets', 'names', 'img', 'x', 'y', 'h', 'w'])98 return odf99def add_key_to_dict(id, collection):100 if id in collection:101 id_obj = collection[id]102 else:103 id_obj = len(collection)104 collection[id] = id_obj105 return id_obj106def extract_knowledge_graph(vg, limit=10, report=2e5):107 section = section_logger(1)108 counter = 0109 synset_ids = {}110 knowledge_graph = nx.DiGraph()111 for img in vg['relationships']:112 if counter > limit:113 break114 else:115 counter += 1116 if counter % report == 0:117 section("Loaded relationships in {} images".format(counter))118 for r in img['relationships']:119 object = r['object']['synsets']120 subject = r['subject']['synsets']121 predicate = r['synsets']122 for syn_obj in object:123 for syn_sub in subject:124 id_obj = add_key_to_dict(syn_obj, synset_ids)125 id_sub = add_key_to_dict(syn_sub, synset_ids)126 if len(predicate) == 1:127 predicate_str = predicate[0]128 else:129 predicate_str = 'relation.n.01'130 knowledge_graph.add_edge(id_obj, id_sub, predicate=predicate_str)131 return knowledge_graph, synset_ids132def extract_data_analytics():133 section = section_logger()134 section('Extracting Visual Genome')135 vg = load_visual_genome()136 section('Generating DataFrames from the object files')137 odf = extract_object_dataframe(vg, int(1e6))138 section('Generating DataFrames from the relationship files')139 rdf = extract_relationship_dataframe(vg, int(1e6))140 section('Saving CSVs')141 odf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_objects.csv'))142 rdf.to_csv(os.path.join(cf.VG_ANALYTICS, 'vg_relationships.csv'))143def extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, relation, predicate):144 hypernyms = wordnet_syn.__getattribute__(relation)()145 for hyp in hypernyms:146 hyp_name = hyp.name()147 if hyp_name in synset_ids:148 hyp_id = synset_ids[hyp_name]149 else:150 hyp_id = add_key_to_dict(hyp_name, synset_ids)151 graph.add_edge(syn_id, hyp_id, predicate=predicate)152 return graph, synset_ids153def add_wordnet_synsets(graph, synset_ids, report=2e5):154 old_synset_ids = dict(synset_ids)155 section = section_logger(1)156 counter = 0157 for syn_name, syn_id in old_synset_ids.items():158 counter += 1159 if counter % report == 0:160 section("Loaded relationships in {} images".format(counter))161 wordnet_syn = wordnet.synset(syn_name)162 extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hypernyms', 'generalize.v.01')163 extract_relations_wordnet(graph, syn_id, synset_ids, wordnet_syn, 'hyponyms', 'specialize.v.01')164 return graph, synset_ids165def save_graph_info(graph, ids):166 nx.write_gml(graph, os.path.join(cf.VG_ANALYTICS, 'knowledge_graph.gml'))167 with open(os.path.join(cf.VG_ANALYTICS, 'synset_ids.json'), 'w') as outfile:168 json.dump(ids, outfile)169def compute_knowledge_graph():170 section = section_logger()171 section('Loading JSON files from dataset')172 vg = load_visual_genome()173 section('Generating Knowledge Graph')174 vi_graph, synset_ids = extract_knowledge_graph(vg, limit=1e10)175 section('Annotating with Wordnet synsets')176 wn_graph, synset_ids = add_wordnet_synsets(vi_graph, synset_ids)177 section('Saving graph')178 save_graph_info(wn_graph, synset_ids)179 return wn_graph180def load_graph_info():181 wn_graph = nx.read_gml(os.path.join(cf.VG_ANALYTICS, 'knowledge_graph.gml'))182 with open(os.path.join(cf.VG_ANALYTICS, 'synset_ids.json'), 'r') as infile:183 ids = json.load(infile)184 return wn_graph, ids...

Full Screen

Full Screen

settings.py

Source:settings.py Github

copy

Full Screen

1"""2Django settings for redbutterfly project.3Generated by 'django-admin startproject' using Django 1.10.4For more information on this file, see5https://docs.djangoproject.com/en/1.10/topics/settings/6For the full list of settings and their values, see7https://docs.djangoproject.com/en/1.10/ref/settings/8"""9import os10import logging11from util.log_util import *12# Build paths inside the project like this: os.path.join(BASE_DIR, ...)13BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))14SECRET_KEY = 'mpjq@%1+hv@j90oxs0spt&9#4xeg0#g_n&9nnb)v_$-oz4!39$'15DEBUG = True16ALLOWED_HOSTS = ['*']17#default ini18DEFAULT_CONFIG_FILE = os.path.join(BASE_DIR, "conf/server.ini")19SECTION_REDIS = 'redis_server'20SECTION_LOGGER = 'server_log'21#logger info define22SERVICE_LOGGERFILE = os.path.join(BASE_DIR, "log/project_log/service.log")23CreatLogger('server', SERVICE_LOGGERFILE, SECTION_LOGGER)24server_logger = logging.getLogger('server')25# Application definition26INSTALLED_APPS = [27 'django.contrib.admin',28 'django.contrib.auth',29 'django.contrib.contenttypes',30 'django.contrib.sessions',31 'django.contrib.messages',32 'django.contrib.staticfiles',33 'rest_framework',34 'red',35]36MIDDLEWARE_CLASSES = [37 'django.middleware.security.SecurityMiddleware',38 'django.contrib.sessions.middleware.SessionMiddleware',39 'django.middleware.locale.LocaleMiddleware',40 'django.middleware.common.CommonMiddleware',41 'django.middleware.csrf.CsrfViewMiddleware',42 'django.contrib.auth.middleware.AuthenticationMiddleware',43 'django.contrib.auth.middleware.SessionAuthenticationMiddleware',44 'django.contrib.messages.middleware.MessageMiddleware',45 'django.middleware.clickjacking.XFrameOptionsMiddleware',46]47ROOT_URLCONF = 'conf.urls'48TEMPLATES = [49 {50 'BACKEND': 'django.template.backends.django.DjangoTemplates',51 'DIRS': [os.path.join(BASE_DIR,'templates')],52 'APP_DIRS': True,53 'OPTIONS': {54 'context_processors': [55 'django.template.context_processors.debug',56 'django.template.context_processors.request',57 'django.contrib.auth.context_processors.auth',58 'django.contrib.messages.context_processors.messages',59 ],60 },61 },62]63TEMPLATE_CONTEXT_PROCESSORS = (64 "django.template.context_processors.request",65 "django.contrib.auth.context_processors.auth",66 "django.template.context_processors.debug",67 "django.template.context_processors.i18n",68 "django.template.context_processors.media",69 "django.template.context_processors.static",70 "django.template.context_processors.tz",71 "django.contrib.messages.context_processors.messages"72)73WSGI_APPLICATION = 'conf.wsgi.application'74DATABASES = {75 'default': {76 'ENGINE': 'django.db.backends.mysql',77 'NAME': 'redbutterfly',78 'USER': 'root',79 'PASSWORD': '426334',80 'HOST': '',81 'PORT': '',82 }83}84CACHES = {85 'default': {86 'BACKEND': 'django_redis.cache.RedisCache',87 'LOCATION': 'redis://127.0.0.1:6379/2',88 'OPTIONS': {89 'CLIENT_CLASS': 'django_redis.client.DefaultClient',90 },91 },92}93REDIS_TIMEOUT=7*24*60*6094CUBES_REDIS_TIMEOUT=60*6095NEVER_REDIS_TIMEOUT=365*24*60*6096LOGIN_URL='/login/'97AUTH_USER_MODEL = 'red.User'98AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend',)99# Password validation100# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators101AUTH_PASSWORD_VALIDATORS = [102 {103 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',104 },105 {106 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',107 },108 {109 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',110 },111 {112 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',113 },114]115# Internationalization116# https://docs.djangoproject.com/en/1.10/topics/i18n/117LANGUAGE_CODE = 'en-us'118TIME_ZONE = 'UTC'119USE_I18N = True120USE_L10N = True121USE_TZ = True122# Static files (CSS, JavaScript, Images)123# https://docs.djangoproject.com/en/1.10/howto/static-files/124STATIC_URL = '/static/'125MEDIA_ROOT ='media/'...

Full Screen

Full Screen

restrictedopd.py

Source:restrictedopd.py Github

copy

Full Screen

...11 obj_df.columns = ['id', 'name', 'number', 'occurrences']12 obj_df = obj_df.astype({'number': 'int32'})13 return obj_df14def extract_distributions(data):15 log = section_logger(1)16 log('Extracting distributions ')17 raw_data = data[['image_id', 'name']].reset_index()18 raw_data.columns = ['class', 'image_id', 'name']19 raw_data = raw_data.astype({'image_id': 'int32'})20 frequencies = raw_data.pivot_table(21 index='image_id',22 columns='name',23 values='class',24 aggfunc='count',25 fill_value=0.0)26 frequencies['sum'] = frequencies.sum(1)27 frequencies[frequencies.columns.difference(['image_id', 'sum'])] = \28 frequencies[frequencies.columns.difference(['image_id', 'sum'])].div(frequencies["sum"], axis=0)29 img_objs = extract_global_objs(frequencies)30 obj_df = extract_objects(data)31 return img_objs, obj_df32def generate_vgopd_from_restrictedvg(output_path, input_path, top_objects, perc):33 section = section_logger()34 section('Loading RestrictedVG Dataset')35 restricted_definitions = load_dataframe(input_path)36 section('Creating distributions')37 image_df, obj_df = extract_distributions(restricted_definitions)38 section('Saving Raw DataFrame')39 save_raw_data(output_path, obj_df, image_df)40 section('Filtering objects')41 data_df = filter_top_objects(image_df, obj_df, top_objects)42 splits = split_distributions(data_df, perc)43 section('Saving final distribution')44 save_distributions(output_path, splits)45if __name__== "__main__":46 generate_vgopd_from_restrictedvg('/home/dani/Documentos/Proyectos/Doctorado/Datasets/RestrictedOPD/1000C',47 '/home/dani/Documentos/Proyectos/Doctorado/Datasets/restrictedGenome', 1000, 0.10)

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run molecule automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful