How to use get_dtypes method in pandera
Best Python code snippet using pandera_python
simfile.py
Source:simfile.py 
...33 # Return output34 return out35####################################################################################################################################36# METHOD TO RETURN DATA TYPES37def get_dtypes():38 """39 Returns a dictionary of data types used40 """41 dtypes={'long' :np.int32,42 'float':np.float32,43 'char' :np.int8}#np.dtype((str,1))}44 return dtypes45####################################################################################################################################46# METHOD TO READ/WRITE FOFCAT FILE47def readgrpcat(fname,grptyp=None,**exkw):48 """49 Reads data from group catelogue files files50 """51 out={} ; dts=get_dtypes()52 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')53 varlist=[dict(name='len' ,dim=1,dtype=dts['long']),54 dict(name='offset',dim=1,dtype=dts['long'])]55 if grptyp=='fof': varlist+=[dict(name='subs' ,dim=1,dtype=dts['long'])]56 elif grptyp=='sub': varlist+=[dict(name='parent',dim=1,dtype=dts['long'])]57 with open(fname,mode='rb') as file: 58 out['Ngrp']=np.fromfile(file,dtype=dts['long'],count=1)[0]59 if out['Ngrp']==0:60 for ivar in varlist: out[ivar['name']]=np.zeros((out['Ngrp']*ivar['dim']),dtype=ivar['dtype'])61 else:62 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Ngrp']*ivar['dim'])63 for ivar in varlist: 64 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))65 return out66####################################################################################################################################67# METHOD TO READ GROUP PROPERTY FILES68def readgrpprop(fname,grptyp=None,**exkw):69 """70 Reads data from group property files71 """72 out={} ; dts=get_dtypes()73 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')74 varlist=[dict(name='cm' ,dim=3,dtype=dts['float']),75 dict(name='cmv' ,dim=3,dtype=dts['float']),76 dict(name='mtot',dim=1,dtype=dts['float']),77 dict(name='mgas',dim=1,dtype=dts['float'])]78 with open(fname,mode='rb') as file: 79 out['Ngrp']=np.fromfile(file,dtype=dts['long'],count=1)[0]80 if out['Ngrp']==0:81 for ivar in varlist: out[ivar['name']]=np.zeros((out['Ngrp']*ivar['dim']),dtype=ivar['dtype'])82 else:83 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Ngrp']*ivar['dim'])84 for ivar in varlist: 85 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))86 return out87####################################################################################################################################88# METHOD TO READ GROUP TYPE FILES89def readgrptype(fname,grptyp=None,**exkw):90 """91 Reads data from group type files92 """93 out={} ; dts=get_dtypes()94 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')95 varlist=[dict(name='type',dim=1,dtype=dts['char'])]96 with open(fname,mode='rb') as file: 97 out['Npart']=np.fromfile(file,dtype=dts['long'],count=1)[0]98 if out['Npart']==0:99 for ivar in varlist: out[ivar['name']]=np.zeros((out['Npart']*ivar['dim']),dtype=ivar['dtype'])100 else:101 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Npart']*ivar['dim'])102 for ivar in varlist: 103 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))104 return out105####################################################################################################################################106# METHOD TO READ GROUP PARTICLE ID FILES107def readgrpids(fname,grptyp=None,**exkw):108 """109 Reads data from group particle ID files110 """111 out={} ; dts=get_dtypes()112 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')113 varlist=[dict(name='ids',dim=1,dtype=dts['long'])]114 with open(fname,mode='rb') as file: 115 out['Npart']=np.fromfile(file,dtype=dts['long'],count=1)[0]116 if out['Npart']==0:117 for ivar in varlist: out[ivar['name']]=np.zeros((out['Npart']*ivar['dim']),dtype=ivar['dtype'])118 else:119 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Npart']*ivar['dim'])120 for ivar in varlist: 121 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))122 return out123####################################################################################################################################124# METHOD TO READ GROUP PARTICLE POSITION FILES125def readgrppos(fname,grptyp=None,**exkw):126 """127 Reads data from group particle position files128 """129 out={} ; dts=get_dtypes()130 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')131 varlist=[dict(name='pos',dim=3,dtype=dts['float'])]132 with open(fname,mode='rb') as file: 133 out['Npart']=np.fromfile(file,dtype=dts['long'],count=1)[0]134 if out['Npart']==0:135 for ivar in varlist: out[ivar['name']]=np.zeros((out['Npart']*ivar['dim']),dtype=ivar['dtype'])136 else:137 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Npart']*ivar['dim'])138 for ivar in varlist: 139 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))140 return out141####################################################################################################################################142# METHOD TO READ GROUP PARTICLE VELOCITY FILES143def readgrpvel(fname,grptyp=None,**exkw):144 """145 Reads data from group particle velocity files146 """147 out={} ; dts=get_dtypes()148 grptyp=mmlpars.mml_pars(grptyp,list=['fof','sub'],default='fof')149 varlist=[dict(name='vel',dim=3,dtype=dts['float'])]150 with open(fname,mode='rb') as file: 151 out['Npart']=np.fromfile(file,dtype=dts['long'],count=1)[0]152 if out['Npart']==0:153 for ivar in varlist: out[ivar['name']]=np.zeros((out['Npart']*ivar['dim']),dtype=ivar['dtype'])154 else:155 for ivar in varlist: out[ivar['name']]=np.fromfile(file,dtype=ivar['dtype'],count=out['Npart']*ivar['dim'])156 for ivar in varlist: 157 if ivar['dim']>1: out[ivar['name']]=out[ivar['name']].reshape((-1,ivar['dim']))158 return out159####################################################################################################################################160# METHOD TO READ/WRITE MTREE TABLES161def rwtable(rwid,fname,method=None,**exkw):...
app.py
Source:app.py
...110 # Segment.generate_manip_options()111 112 form = ManipulateForm()113 114 fields = Segments[session['id']].get_dtypes()115 if 'btn_manip_normalize' in request.form.keys():116 col = request.form['select-variable']117 Segments[session['id']].manip_num_normalize(col)118 fields = Segments[session['id']].get_dtypes()119 return render_template(120 'manipulate.html',121 variables=fields,122 manipulations=Segments[session['id']]._manipulations,123 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),124 dfshape=Segments[session['id']].df.shape,125 form=form126 )127 if 'btn_manip_trim' in request.form.keys():128 col = request.form['select-variable']129 lower = int(request.form['manip_trim_lower'])130 upper = int(request.form['manip_trim_upper'])131 Segments[session['id']].manip_num_trim(col, upper, lower)132 fields = Segments[session['id']].get_dtypes()133 return render_template(134 'manipulate.html',135 variables=fields,136 manipulations=Segments[session['id']]._manipulations,137 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),138 dfshape=Segments[session['id']].df.shape,139 form=form140 )141 if 'btn_manip_bin' in request.form.keys():142 d = request.form.to_dict()143 d = {144 re.sub('manip_bin_','',k): v 145 for k,v in d.items() if k.startswith('manip_bin')146 }147 Segments[session['id']].manip_cat_bins(request.form.get('select-variable'), d)148 fields = Segments[session['id']].get_dtypes()149 Segments[session['id']].generate_manip_options() # Regenerate the manip options as cats will have changed150 return render_template(151 'manipulate.html',152 variables=fields,153 manipulations=Segments[session['id']]._manipulations,154 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),155 dfshape=Segments[session['id']].df.shape,156 form=form157 )158 159 if 'btn_manip_datediff' in request.form.keys():160 col = request.form['select-variable']161 datediff = request.form['manip_datediff']162 Segments[session['id']].manip_dt_datediff(col, datediff)163 fields = Segments[session['id']].get_dtypes()164 return render_template(165 'manipulate.html',166 variables=fields,167 manipulations=Segments[session['id']]._manipulations,168 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),169 dfshape=Segments[session['id']].df.shape,170 form=form171 )172 173 if 'btn_manip_emb' in request.form.keys():174 col = request.form['select-variable']175 Segments[session['id']].manip_str_embeddings(col)176 fields = Segments[session['id']].get_dtypes()177 return render_template(178 'manipulate.html',179 variables=fields,180 manipulations=Segments[session['id']]._manipulations,181 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),182 dfshape=Segments[session['id']].df.shape,183 form=form184 )185 if 'btn_manipulate_to_visualise' in request.form.keys():186 return redirect('/visualise')187 return render_template(188 'manipulate.html',189 variables=fields,190 manipulations=Segments[session['id']]._manipulations,191 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),192 dfshape=Segments[session['id']].df.shape,193 form=form194 )195@app.route("/visualise", methods=["GET","POST"])196def visualise():197 form = ManipulateForm()198 if 'btn_viz_to_cluster' in request.form.keys():199 return redirect('/cluster')200 graphJSON = generate_graph(request.args, Segments[session['id']].df)201 return render_template(202 'visualise.html',203 graphJSON=graphJSON,204 dtypes=Segments[session['id']].get_dtypes(),205 dfshape=Segments[session['id']].df.shape,206 form=form207 )208@app.route("/callback", methods=["GET","POST"])209def cb():210 return generate_graph(request.args, Segments[session['id']].df)211@app.route("/cluster", methods=["GET","POST"])212def cluster():213 # Segment.df = pd.read_csv('/home/george/Development/Personal/Python/Segment/data/processed.csv')214 # Segment.edit_to_cat('Cat')215 # Segment.edit_to_datetime('FirstOrder')216 # Segment.edit_to_str('CustomerID')217 218 # Segment.manip_cat_bins('Cat',{'A':'A','B':'B','C':'C','D':'E'})219 # Segment.manip_dt_datediff('FirstOrder','2010-12-01')220 # Segment.manip_num_normalize('Num_Orders')221 # Segment.manip_num_trim('Num_Orders',15,0)222 # Segment.manip_str_embeddings('Description')223 # Segment.generate_manip_options()224 form = ClusterForm()225 if form.validate_on_submit():226 if 'btn_cluster_to_clusterviz' in request.form.keys():227 return redirect('/clusterviz')228 return render_template(229 'cluster.html',230 dtypes=Segments[session['id']].get_dtypes(),231 clustered=Segments[session['id']]._is_clustered,232 dfshape=Segments[session['id']].df.shape,233 html_table=create_html_output(Segments[session['id']].df, DIV_SIZE),234 form=form235 )236@app.route("/cluster-data", methods=["GET","POST","PUT"])237def cluster_data():238 data = request.get_json()239 tsne_comps = 3 if data['tsne'] else None240 Segments[session['id']]._cluster_fields = data['fields']241 Segments[session['id']].cluster(Segments[session['id']]._cluster_fields, n_components=tsne_comps)242 return "ok!"243@app.route("/clusterviz", methods=["GET","POST"])244def clusterviz():245 246 form = ManipulateForm()247 graphJSON = generate_graph(request.args, Segments[session['id']].df)248 return render_template(249 'cluster_viz.html',250 graphJSON=graphJSON,251 dtypes=Segments[session['id']].get_dtypes(),252 clustered=Segments[session['id']]._is_clustered,253 dfshape=Segments[session['id']].df.shape,254 form=form255 )256if __name__ == '__main__':...
load_data.py
Source:load_data.py
1import scipy.io2import numpy as np3def load_data(filename):4 arr = scipy.io.loadmat(filename)['data']5 dtypes = get_dtypes(arr)6 output = np.zeros([], dtype=dtypes)7 copy_values(arr, output)8 return output.view(np.recarray)9def get_dtypes(arr):10 names = arr.dtype.names11 dtypes = []12 shapes = []13 for name in names:14 sub_names = arr[name][0][0].dtype.names15 if sub_names:16 dtypes.append(get_dtypes(arr[name][0][0]))17 else:18 dtypes.append(arr[name][0][0].dtype)19 shapes.append(arr[name][0][0].squeeze().shape)20 return {'names': names, 'formats': zip(dtypes, shapes)}21def copy_values(copy_from, copy_to):22 names = copy_to.dtype.names23 for name in names:24 if isinstance(copy_to[name], (np.ndarray, np.record)) and copy_to[name].dtype.names:25 copy_values(copy_from[name][0][0], copy_to[name])26 else:27 copy_to[name] = copy_from[name][0][0].squeeze()28def test_load_data():29 data = load_data('/Volumes/rhino/data/eeg/scalp/ltp/ltpFR/behavioral/data/stat_data_LTP063.mat')30 from spc.view_recarray import pprint_rec as ppr...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run pandera automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
