How to use rename_columns method in pandera

Best Python code snippet using pandera_python

Chassis_Dashboard_Automated.py

Source:Chassis_Dashboard_Automated.py

...131132all_chassis_total_wty,all_chassis_cpc_cpu, all_chassis_top_customers,all_chassis_top_dealers, all_chassis_seag_info = get_final_files(st_date, ed_date, data_for_active_units, chassis_claims, "All_chassis")133  134all_chassis_top_dealers.reset_index(inplace = True)135all_chassis_total_wty = rename_columns('total_wty', all_chassis_total_wty)136all_chassis_cpc_cpu = rename_columns('cpc_cpu', all_chassis_cpc_cpu)137all_chassis_top_customers = rename_columns('top_customers', all_chassis_top_customers)138all_chassis_top_dealers = rename_columns('top_dealers', all_chassis_top_dealers)139140all_chassis_cpc_cpu['CPC'] = all_chassis_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)141all_chassis_cpc_cpu['CPU'] = all_chassis_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)142all_chassis  = pd.concat([all_chassis_total_wty, all_chassis_cpc_cpu, all_chassis_top_customers,all_chassis_top_dealers,all_chassis_seag_info], axis =1)143all_chassis.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/All_chassis.csv', index = False)144 145p4_chassis_total_wty,p4_chassis_cpc_cpu, p4_chassis_top_customers,p4_chassis_top_dealers, p4_chassis_seag_info = get_final_files(st_date, ed_date, p4_contracts, p4_claims, "P4_chassis")146  147p4_chassis_top_dealers.reset_index(inplace = True)148p4_chassis_total_wty = rename_columns('total_wty', p4_chassis_total_wty)149p4_chassis_cpc_cpu = rename_columns('cpc_cpu', p4_chassis_cpc_cpu)150p4_chassis_top_customers = rename_columns('top_customers', p4_chassis_top_customers)151p4_chassis_top_dealers = rename_columns('top_dealers', p4_chassis_top_dealers)152153p4_chassis_cpc_cpu['CPC'] = p4_chassis_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)154p4_chassis_cpc_cpu['CPU'] = p4_chassis_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)155p4_chassis  = pd.concat([p4_chassis_total_wty, p4_chassis_cpc_cpu, p4_chassis_top_customers,p4_chassis_top_dealers,p4_chassis_seag_info], axis =1)156p4_chassis.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/p4_chassis.csv', index = False)157158159seal_oil_total_wty,seal_oil_cpc_cpu, seal_oil_top_customers,seal_oil_top_dealers, seal_oil_seag_info = get_final_files(st_date, ed_date, seal_oil_contracts, seal_oil_claims, "seal_oil")160seal_oil_top_dealers.reset_index(inplace = True)161seal_oil_total_wty = rename_columns('total_wty', seal_oil_total_wty)162seal_oil_cpc_cpu = rename_columns('cpc_cpu', seal_oil_cpc_cpu)163seal_oil_top_customers = rename_columns('top_customers', seal_oil_top_customers)164seal_oil_top_dealers = rename_columns('top_dealers', seal_oil_top_dealers)165166seal_oil_paid_amt_bld_yr, seal_oil_total_claims_fail_miles_isy ,seal_oil_paid_amt_mis = get_secondary_data(seal_oil_claims)167seal_oil_paid_amt_bld_yr = rename_columns('bld_yr', seal_oil_paid_amt_bld_yr)168seal_oil_total_claims_fail_miles_isy = rename_columns('fail_miles', seal_oil_total_claims_fail_miles_isy)169seal_oil_paid_amt_mis = rename_columns('mis', seal_oil_paid_amt_mis)170171seal_oil_cpc_cpu['CPC'] = seal_oil_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)172seal_oil_cpc_cpu['CPU'] = seal_oil_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)173seal_oil  = pd.concat([seal_oil_total_wty, seal_oil_cpc_cpu, seal_oil_top_customers,seal_oil_top_dealers,seal_oil_seag_info, seal_oil_paid_amt_bld_yr,seal_oil_total_claims_fail_miles_isy,seal_oil_paid_amt_mis], axis =1)174seal_oil.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/seal_oil.csv', index = False)175176177mirror_heated_convex_total_wty,mirror_heated_convex_cpc_cpu, mirror_heated_convex_top_customers,mirror_heated_convex_top_dealers, mirror_heated_convex_seag_info = get_final_files(st_date, ed_date, mirror_heated_convex_contracts, mirror_heated_convex_claims, "mirror_heated_convex")178mirror_heated_convex_top_dealers.reset_index(inplace = True)179mirror_heated_convex_total_wty = rename_columns('total_wty', mirror_heated_convex_total_wty)180mirror_heated_convex_cpc_cpu = rename_columns('cpc_cpu', mirror_heated_convex_cpc_cpu)181mirror_heated_convex_top_customers = rename_columns('top_customers', mirror_heated_convex_top_customers)182mirror_heated_convex_top_dealers = rename_columns('top_dealers', mirror_heated_convex_top_dealers)183184mirror_heated_convex_paid_amt_bld_yr, mirror_heated_convex_total_claims_fail_miles_isy ,mirror_heated_convex_paid_amt_mis = get_secondary_data(mirror_heated_convex_claims)185mirror_heated_convex_paid_amt_bld_yr = rename_columns('bld_yr', mirror_heated_convex_paid_amt_bld_yr)186mirror_heated_convex_total_claims_fail_miles_isy = rename_columns('fail_miles', mirror_heated_convex_total_claims_fail_miles_isy)187mirror_heated_convex_paid_amt_mis = rename_columns('mis', mirror_heated_convex_paid_amt_mis)188189mirror_heated_convex_cpc_cpu['CPC'] = mirror_heated_convex_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)190mirror_heated_convex_cpc_cpu['CPU'] = mirror_heated_convex_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)191mirror_heated_convex  = pd.concat([mirror_heated_convex_total_wty, mirror_heated_convex_cpc_cpu, mirror_heated_convex_top_customers,mirror_heated_convex_top_dealers,mirror_heated_convex_seag_info, mirror_heated_convex_paid_amt_bld_yr,mirror_heated_convex_total_claims_fail_miles_isy,mirror_heated_convex_paid_amt_mis], axis =1)192mirror_heated_convex.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/mirror_heated_convex.csv', index = False)193194195core_tank_radiator_total_wty,core_tank_radiator_cpc_cpu, core_tank_radiator_top_customers,core_tank_radiator_top_dealers, core_tank_radiator_seag_info = get_final_files(st_date, ed_date, core_tank_radiator_contracts, core_tank_radiator_claims, "core_tank_radiator")196core_tank_radiator_top_dealers.reset_index(inplace = True)197core_tank_radiator_total_wty = rename_columns('total_wty', core_tank_radiator_total_wty)198core_tank_radiator_cpc_cpu = rename_columns('cpc_cpu', core_tank_radiator_cpc_cpu)199core_tank_radiator_top_customers = rename_columns('top_customers', core_tank_radiator_top_customers)200core_tank_radiator_top_dealers = rename_columns('top_dealers', core_tank_radiator_top_dealers)201202core_tank_radiator_paid_amt_bld_yr, core_tank_radiator_total_claims_fail_miles_isy ,core_tank_radiator_paid_amt_mis = get_secondary_data(core_tank_radiator_claims)203core_tank_radiator_paid_amt_bld_yr = rename_columns('bld_yr', core_tank_radiator_paid_amt_bld_yr)204core_tank_radiator_total_claims_fail_miles_isy = rename_columns('fail_miles', core_tank_radiator_total_claims_fail_miles_isy)205core_tank_radiator_paid_amt_mis = rename_columns('mis', core_tank_radiator_paid_amt_mis)206207core_tank_radiator_cpc_cpu['CPC'] = core_tank_radiator_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)208core_tank_radiator_cpc_cpu['CPU'] = core_tank_radiator_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)209core_tank_radiator  = pd.concat([core_tank_radiator_total_wty, core_tank_radiator_cpc_cpu, core_tank_radiator_top_customers,core_tank_radiator_top_dealers,core_tank_radiator_seag_info, core_tank_radiator_paid_amt_bld_yr,core_tank_radiator_total_claims_fail_miles_isy,core_tank_radiator_paid_amt_mis], axis =1)210core_tank_radiator.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/core_tank_radiator.csv', index = False)211212213wiring_harness_total_wty,wiring_harness_cpc_cpu, wiring_harness_top_customers,wiring_harness_top_dealers, wiring_harness_seag_info = get_final_files(st_date, ed_date, wiring_harness_contracts, wiring_harness_claims, "wiring_harness")214wiring_harness_top_dealers.reset_index(inplace = True)215wiring_harness_total_wty = rename_columns('total_wty', wiring_harness_total_wty)216wiring_harness_cpc_cpu = rename_columns('cpc_cpu', wiring_harness_cpc_cpu)217wiring_harness_top_customers = rename_columns('top_customers', wiring_harness_top_customers)218wiring_harness_top_dealers = rename_columns('top_dealers', wiring_harness_top_dealers)219220wiring_harness_paid_amt_bld_yr, wiring_harness_total_claims_fail_miles_isy ,wiring_harness_paid_amt_mis = get_secondary_data(wiring_harness_claims)221wiring_harness_paid_amt_bld_yr = rename_columns('bld_yr', wiring_harness_paid_amt_bld_yr)222wiring_harness_total_claims_fail_miles_isy = rename_columns('fail_miles', wiring_harness_total_claims_fail_miles_isy)223wiring_harness_paid_amt_mis = rename_columns('mis', wiring_harness_paid_amt_mis)224225wiring_harness_cpc_cpu['CPC'] = wiring_harness_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)226wiring_harness_cpc_cpu['CPU'] = wiring_harness_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)227wiring_harness  = pd.concat([wiring_harness_total_wty, wiring_harness_cpc_cpu, wiring_harness_top_customers,wiring_harness_top_dealers,wiring_harness_seag_info, wiring_harness_paid_amt_bld_yr,wiring_harness_total_claims_fail_miles_isy,wiring_harness_paid_amt_mis], axis =1)228wiring_harness.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/wiring_harness.csv', index = False)229230 231heater_aux_total_wty,heater_aux_cpc_cpu, heater_aux_top_customers,heater_aux_top_dealers, heater_aux_seag_info = get_final_files(st_date, ed_date, heater_aux_contracts, heater_aux_claims, "heater_aux")232heater_aux_top_dealers.reset_index(inplace = True)233heater_aux_total_wty = rename_columns('total_wty', heater_aux_total_wty)234heater_aux_cpc_cpu = rename_columns('cpc_cpu', heater_aux_cpc_cpu)235heater_aux_top_customers = rename_columns('top_customers', heater_aux_top_customers)236heater_aux_top_dealers = rename_columns('top_dealers', heater_aux_top_dealers)237238heater_aux_paid_amt_bld_yr, heater_aux_total_claims_fail_miles_isy ,heater_aux_paid_amt_mis = get_secondary_data(heater_aux_claims)239heater_aux_paid_amt_bld_yr = rename_columns('bld_yr', heater_aux_paid_amt_bld_yr)240heater_aux_total_claims_fail_miles_isy = rename_columns('fail_miles', heater_aux_total_claims_fail_miles_isy)241heater_aux_paid_amt_mis = rename_columns('mis', heater_aux_paid_amt_mis)242243heater_aux_cpc_cpu['CPC'] = heater_aux_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)244heater_aux_cpc_cpu['CPU'] = heater_aux_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)245heater_aux  = pd.concat([heater_aux_total_wty, heater_aux_cpc_cpu, heater_aux_top_customers,heater_aux_top_dealers,heater_aux_seag_info, heater_aux_paid_amt_bld_yr,heater_aux_total_claims_fail_miles_isy,heater_aux_paid_amt_mis], axis =1)246heater_aux.to_csv(r'Z:/2000_Cost Management/2500_General Information/2502_Processes & Documentation/Tableau Documentation/All_dashboards_combined/heater_aux.csv', index = False)247248ac_compressor_total_wty,ac_compressor_cpc_cpu, ac_compressor_top_customers,ac_compressor_top_dealers, ac_compressor_seag_info = get_final_files(st_date, ed_date, ac_compressor_contracts, ac_compressor_claims, "ac_compressor")249ac_compressor_top_dealers.reset_index(inplace = True)250ac_compressor_total_wty = rename_columns('total_wty', ac_compressor_total_wty)251ac_compressor_cpc_cpu = rename_columns('cpc_cpu', ac_compressor_cpc_cpu)252ac_compressor_top_customers = rename_columns('top_customers', ac_compressor_top_customers)253ac_compressor_top_dealers = rename_columns('top_dealers', ac_compressor_top_dealers)254255ac_compressor_paid_amt_bld_yr, ac_compressor_total_claims_fail_miles_isy ,ac_compressor_paid_amt_mis = get_secondary_data(ac_compressor_claims)256ac_compressor_paid_amt_bld_yr = rename_columns('bld_yr', ac_compressor_paid_amt_bld_yr)257ac_compressor_total_claims_fail_miles_isy = rename_columns('fail_miles', ac_compressor_total_claims_fail_miles_isy)258ac_compressor_paid_amt_mis = rename_columns('mis', ac_compressor_paid_amt_mis)259260ac_compressor_cpc_cpu['CPC'] = ac_compressor_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Total_Claims_cpc_cpu'] if x['Total_Claims_cpc_cpu'] > 0 else 0, axis =1)261ac_compressor_cpc_cpu['CPU'] = ac_compressor_cpc_cpu.apply(lambda x: x['Total_Cost_cpc_cpu']/x['Active_Units'] if x['Active_Units'] >0 else 0, axis =1)262ac_compressor  = pd.concat([ac_compressor_total_wty, ac_compressor_cpc_cpu, ac_compressor_top_customers,ac_compressor_top_dealers,ac_compressor_seag_info, ac_compressor_paid_amt_bld_yr,ac_compressor_total_claims_fail_miles_isy,ac_compressor_paid_amt_mis], axis =1)
...

reader.py

Source:reader.py

1from io import StringIO2import numpy as np3import pandas as pd4class TableDef:5    def __init__(self, table_id, end_table_tag, skip_rows=0, widths=None,6                 df_drop_top_rows=None, df_drop_tail_rows=None, reset_header=False,7                 rename_columns=None, convert_numerics=False, int_columns=None,8                 index_col=None9                 ):10        self.table_id = table_id11        self.end_table_tag = end_table_tag12        self.skip_rows = skip_rows13        self.widths = widths14        self.df_drop_top_rows = df_drop_top_rows15        self.df_drop_tail_rows = df_drop_tail_rows16        self.reset_header = reset_header17        self.rename_columns = rename_columns18        self.convert_numerics = convert_numerics19        self.int_columns = int_columns20        self.index_col = index_col21_table_parameters = {22    '1.02': TableDef('1.02', ' Adjustment of', skip_rows=3, df_drop_top_rows=1,23                     df_drop_tail_rows=4, reset_header=False,24                     rename_columns=None, convert_numerics=False, 25                     int_columns=['BoardCount'],26                     index_col=None),27    '2.04': TableDef('2.04', 'COUNT', skip_rows=10, df_drop_top_rows=1,28                     df_drop_tail_rows=2, rename_columns={'Unnamed: 0': 'origin'},29                     index_col = 'origin', convert_numerics=True,30                     ),31    '2.05': TableDef('2.05',end_table_tag='-----------', skip_rows=5,32                     df_drop_top_rows=1,df_drop_tail_rows=1,33                     rename_columns={'Unnamed: 0': 'origin'}, index_col='origin',34                     convert_numerics=True),35    '2.07': TableDef('2.07', end_table_tag='-----------', skip_rows=8, # 'df_drop_top_rows': 1,36                     df_drop_tail_rows=1, reset_header=True, widths=[7, 11, 11, 11, 11],37                     rename_columns={38                        0: 'station_group',39                        1: 'pre_calib_board',40                        2: 'station_count',41                        3: 'station_target',42                        4: 'post_calib_board'43                    },44                    convert_numerics=True,int_columns='station_group', index_col='station_group'),45    '2.08': TableDef('2.08', end_table_tag='Number of unique', skip_rows=8, # 'df_drop_top_rows': 1,46                     df_drop_tail_rows=1, reset_header=True, widths=[6, 22, 11, 11, 11, 10],47                     rename_columns={48                        0: 'route_group_num',49                        1: 'route_group',50                        2: 'pre_calib_board',51                        3: 'route_count',52                        4: 'route_target',53                        5: 'post_calib_board'54                    }, convert_numerics=True, #'int_columns': 'station_group',55                    index_col='route_group_num'),56    '3.01': TableDef('3.01', end_table_tag='TOTAL', skip_rows=8,57                     df_drop_top_rows=1, index_col='origin',58                     rename_columns={'Unnamed: 0': 'origin'}),59    '3.02': TableDef('3.02', end_table_tag='TOTAL', skip_rows=8,60                     df_drop_top_rows=1, index_col='origin',61                     rename_columns={'Unnamed: 0': 'origin'}),62    '3.03': TableDef('3.03', end_table_tag='TOTAL', skip_rows=8,63                     df_drop_top_rows=1, index_col='origin',64                     rename_columns={'Unnamed: 0': 'origin'}),65    '4.01': TableDef('4.01', end_table_tag='Total', skip_rows=5,66                    df_drop_top_rows=1,index_col='origin',67                    rename_columns={'Idist': 'origin'}),68                    69    '4.02': TableDef('4.02', end_table_tag='Total', skip_rows=5,70                    df_drop_top_rows=1,index_col='origin',71                    rename_columns={'Idist': 'origin'}),72    73    '4.04': TableDef('4.04', end_table_tag='Total', skip_rows=5,74                    df_drop_top_rows=1,75                    rename_columns={'Unnamed: 0': 'origin'}),76    '8.01': TableDef('8.01', end_table_tag='Total', skip_rows=5,77                     df_drop_top_rows=1, index_col='origin',78                     rename_columns={'Idist': 'origin'},79                     convert_numerics=True80                     ),81    # Stop Level Boardings82    '9.01': TableDef('9.01', end_table_tag='\x00', skip_rows=8,83                     reset_header=True,84                     rename_columns={85                         0: 'stop_id', 1: 'station_name',86                         2: 'exist_wlk', 3: 'exist_knr', 4: 'exist_pnr', 5: 'exist_xfr', 6: 'exist_all',87                         7: 'nb_wlk', 8: 'nb_knr', 9: 'nb_pnr', 10: 'nb_xfr', 11: 'nb_all',88                         12: 'bld_wlk', 13: 'bld_knr', 14: 'bld_pnr', 15: 'bld_xfr', 16: 'bld_all',89                     },90                     int_columns=['exist_wlk','exist_knr', 'exist_pnr', 'exist_xfr', 'exist_all',91                                  'nb_wlk', 'nb_knr', 'nb_pnr', 'nb_xfr', 'nb_all',92                                  'bld_wlk', 'bld_knr', 'bld_pnr', 'bld_xfr', 'bld_all']93                     ),94    # Route Ridership95    '10.01': TableDef('10.01', end_table_tag='               Total', skip_rows=7,96                      reset_header=True, widths=[25, 30] + [10] * 13, df_drop_top_rows=1,97                      rename_columns={98                          0: 'route_id', 1: 'route_name', 2: 'route_count',99                          3: 'exist_wlk', 4: 'exist_knr', 5: 'exist_pnr', 6: 'exist_all',100                          7: 'nb_wlk', 8: 'nb_knr', 9: 'nb_pnr', 10: 'nb_all',101                          11: 'bld_wlk', 12: 'bld_knr', 13: 'bld_pnr', 14: 'bld_all',102                      },103                      int_columns=[104                          'route_count',105                          'exist_wlk', 'exist_knr', 'exist_pnr', 'exist_all',106                          'nb_wlk', 'nb_knr', 'nb_pnr', 'nb_all',107                          'bld_wlk', 'bld_knr', 'bld_pnr', 'bld_all',108                      ]),109    #Existing - HBW - TRN - 0 Car110    '30.01': TableDef('30.01', end_table_tag='Total', skip_rows=5,111                       df_drop_top_rows=1, index_col='origin',112                       rename_columns={'Idist': 'origin'}, convert_numerics=True),113    114    #Existing - HBW - TRN - 1 Car115    '51.01': TableDef('51.01', end_table_tag='Total', skip_rows=5,116                       df_drop_top_rows=1, index_col='origin',117                       rename_columns={'Idist': 'origin'}, convert_numerics=True),118    119    #Existing - HBW - TRN - 2 Car120    '72.01': TableDef('72.01', end_table_tag='Total', skip_rows=5,121                       df_drop_top_rows=1, index_col='origin',122                       rename_columns={'Idist': 'origin'}, convert_numerics=True),123                       124    #Existing - HBW - TRN - ALL125    '93.01': TableDef('93.01', end_table_tag='Total', skip_rows=5,126                       df_drop_top_rows=1, index_col='origin',127                       rename_columns={'Idist': 'origin'}, convert_numerics=True),128    #Existing - HBO - TRN - 0 Car129    '114.01': TableDef('114.01', end_table_tag='Total', skip_rows=5,130                       df_drop_top_rows=1, index_col='origin',131                       rename_columns={'Idist': 'origin'}, convert_numerics=True),132    #Existing - HBO - TRN - 1 Car133    '135.01': TableDef('135.01', end_table_tag='Total', skip_rows=5,134                       df_drop_top_rows=1, index_col='origin',135                       rename_columns={'Idist': 'origin'}, convert_numerics=True),136    137    #Existing - HBO - TRN - 2 Car138    '156.01': TableDef('156.01', end_table_tag='Total', skip_rows=5,139                       df_drop_top_rows=1, index_col='origin',140                       rename_columns={'Idist': 'origin'}, convert_numerics=True),141    #Existing - HBO - TRN - ALL142    '177.01': TableDef('177.01', end_table_tag='Total', skip_rows=5,143                       df_drop_top_rows=1, index_col='origin',144                       rename_columns={'Idist': 'origin'}, convert_numerics=True),145    #Existing - NHB - TRN - 0 Car146    '198.01': TableDef('198.01', end_table_tag='Total', skip_rows=5,147                       df_drop_top_rows=1, index_col='origin',148                       rename_columns={'Idist': 'origin'}, convert_numerics=True),149    150    #Existing - NHB - TRN - 1 Car151    '219.01': TableDef('219.01', end_table_tag='Total', skip_rows=5,152                       df_drop_top_rows=1, index_col='origin',153                       rename_columns={'Idist': 'origin'}, convert_numerics=True),154    155    #Existing - NHB - TRN - 2 Car156    '240.01': TableDef('240.01', end_table_tag='Total', skip_rows=5,157                       df_drop_top_rows=1, index_col='origin',158                       rename_columns={'Idist': 'origin'}, convert_numerics=True),159                       160    #Existing - NHB - TRN - ALL161    '261.01': TableDef('261.01', end_table_tag='Total', skip_rows=5,162                       df_drop_top_rows=1, index_col='origin',163                       rename_columns={'Idist': 'origin'}, convert_numerics=True),164    #Existing - All Trips - TRN - 0 Car165    '282.01': TableDef('282.01', end_table_tag='Total', skip_rows=5,166                       df_drop_top_rows=1, index_col='origin',167                       rename_columns={'Idist': 'origin'}, convert_numerics=True),168                       169    #Existing - All Trips - TRN - 1 Car170    '303.01': TableDef('303.01', end_table_tag='Total', skip_rows=5,171                       df_drop_top_rows=1, index_col='origin',172                       rename_columns={'Idist': 'origin'}, convert_numerics=True),173                       174    #Existing - All Trips - TRN - 2 Car175    '324.01': TableDef('324.01', end_table_tag='Total', skip_rows=5,176                       df_drop_top_rows=1, index_col='origin',177                       rename_columns={'Idist': 'origin'}, convert_numerics=True),178                179    #Existing - All Trips - TRN - ALL180    '345.01': TableDef('345.01', end_table_tag='Total', skip_rows=5,181                       df_drop_top_rows=1, index_col='origin',182                       rename_columns={'Idist': 'origin'}, convert_numerics=True),183    # No Build - HBW - TRN - 0 Car184    '366.01': TableDef('366.01', end_table_tag='Total', skip_rows=5,185                       df_drop_top_rows=1, index_col='origin',186                       rename_columns={'Idist': 'origin'}, convert_numerics=True),187    188    # No Build - HBW - TRN - 1 Car189    '387.01': TableDef('387.01', end_table_tag='Total', skip_rows=5,190                       df_drop_top_rows=1, index_col='origin',191                       rename_columns={'Idist': 'origin'}, convert_numerics=True),192    193    # No Build - HBW - TRN - 2 Car194    '408.01': TableDef('408.01', end_table_tag='Total', skip_rows=5,195                       df_drop_top_rows=1, index_col='origin',196                       rename_columns={'Idist': 'origin'}, convert_numerics=True),197    # No Build - HBW - TRN - ALL198    '429.01': TableDef('429.01', end_table_tag='Total', skip_rows=5,199                      df_drop_top_rows=1, index_col='origin',200                      rename_columns={'Idist': 'origin'}, convert_numerics=True),201    # No Build - HBO - TRN - O Car202    '450.01': TableDef('450.01', end_table_tag='Total', skip_rows=5,203                       df_drop_top_rows=1, index_col='origin',204                       rename_columns={'Idist': 'origin'}, convert_numerics=True),205    206    # No Build - HBO - TRN - 1 Car207    '471.01': TableDef('471.01', end_table_tag='Total', skip_rows=5,208                       df_drop_top_rows=1, index_col='origin',209                       rename_columns={'Idist': 'origin'}, convert_numerics=True),210    211    # No Build - HBO - TRN - 2 Car212    '492.01': TableDef('492.01', end_table_tag='Total', skip_rows=5,213                       df_drop_top_rows=1, index_col='origin',214                       rename_columns={'Idist': 'origin'}, convert_numerics=True),215    216    # No Build - HBO - TRN - ALL217    '513.01': TableDef('513.01', end_table_tag='Total', skip_rows=5,218                       df_drop_top_rows=1, index_col='origin',219                       rename_columns={'Idist': 'origin'}, convert_numerics=True),220    # No Build - NHB - TRN - 0 Car221    '534.01': TableDef('534.01', end_table_tag='Total', skip_rows=5,222                       df_drop_top_rows=1, index_col='origin',223                       rename_columns={'Idist': 'origin'}, convert_numerics=True),224    # No Build - NHB - TRN - 1 Car225    '555.01': TableDef('555.01', end_table_tag='Total', skip_rows=5,226                       df_drop_top_rows=1, index_col='origin',227                       rename_columns={'Idist': 'origin'}, convert_numerics=True),228    # No Build - NHB - TRN - 2 Car229    '576.01': TableDef('576.01', end_table_tag='Total', skip_rows=5,230                       df_drop_top_rows=1, index_col='origin',231                       rename_columns={'Idist': 'origin'}, convert_numerics=True),232    233    # No Build - NHB - TRN - ALL234    '597.01': TableDef('597.01', end_table_tag='Total', skip_rows=5,235                       df_drop_top_rows=1, index_col='origin',236                       rename_columns={'Idist': 'origin'}, convert_numerics=True),237    # No Build - All Trips - TRN - 0 Car238    '618.01': TableDef('618.01', end_table_tag='Total', skip_rows=5,239                       df_drop_top_rows=1, index_col='origin',240                       rename_columns={'Idist': 'origin'}, convert_numerics=True),241    # No Build - All Trips - TRN - 1 Car242    '639.01': TableDef('639.01', end_table_tag='Total', skip_rows=5,243                       df_drop_top_rows=1, index_col='origin',244                       rename_columns={'Idist': 'origin'}, convert_numerics=True),245    # No Build - All Trips - TRN - 2 Car246    '660.01': TableDef('660.01', end_table_tag='Total', skip_rows=5,247                       df_drop_top_rows=1, index_col='origin',248                       rename_columns={'Idist': 'origin'}, convert_numerics=True),249    # No Build - All Trips - TRN - ALL250    '681.01': TableDef('681.01', end_table_tag='Total', skip_rows=5,251                       df_drop_top_rows=1, index_col='origin',252                       rename_columns={'Idist': 'origin'}, convert_numerics=True),253    # Build - HBW - TRN - 0 Car254    '702.01': TableDef('702.01', end_table_tag='Total', skip_rows=5,255                       df_drop_top_rows=1, index_col='origin',256                       rename_columns={'Idist': 'origin'}, convert_numerics=True),257    # Build - HBW - TRN - 1 Car258    '723.01': TableDef('723.01', end_table_tag='Total', skip_rows=5,259                       df_drop_top_rows=1, index_col='origin',260                       rename_columns={'Idist': 'origin'}, convert_numerics=True),261    # Build - HBW - TRN - 2 Car262    '744.01': TableDef('744.01', end_table_tag='Total', skip_rows=5,263                       df_drop_top_rows=1, index_col='origin',264                       rename_columns={'Idist': 'origin'}, convert_numerics=True),265    266    # Build - HBW - TRN - ALL267    '765.01': TableDef('765.01', end_table_tag='Total', skip_rows=5,268                      df_drop_top_rows=1, index_col='origin',269                      rename_columns={'Idist': 'origin'}, convert_numerics=True),270    # Build - HBO - TRN - 0 Car271    '786.01': TableDef('786.01', end_table_tag='Total', skip_rows=5,272                       df_drop_top_rows=1, index_col='origin',273                       rename_columns={'Idist': 'origin'}, convert_numerics=True),274    # Build - HBO - TRN - 1 Car275    '807.01': TableDef('807.01', end_table_tag='Total', skip_rows=5,276                       df_drop_top_rows=1, index_col='origin',277                       rename_columns={'Idist': 'origin'}, convert_numerics=True),278    # Build - HBO - TRN - 2 Car279    '828.01': TableDef('828.01', end_table_tag='Total', skip_rows=5,280                       df_drop_top_rows=1, index_col='origin',281                       rename_columns={'Idist': 'origin'}, convert_numerics=True),282    # Build - HBO - TRN - ALL283    '849.01': TableDef('849.01', end_table_tag='Total', skip_rows=5,284                       df_drop_top_rows=1, index_col='origin',285                       rename_columns={'Idist': 'origin'}, convert_numerics=True),286    # Build - NHB - TRN - 0 Car287    '870.01': TableDef('870.01', end_table_tag='Total', skip_rows=5,288                       df_drop_top_rows=1, index_col='origin',289                       rename_columns={'Idist': 'origin'}, convert_numerics=True),290    # Build - NHB - TRN - 1 Car291    '891.01': TableDef('891.01', end_table_tag='Total', skip_rows=5,292                       df_drop_top_rows=1, index_col='origin',293                       rename_columns={'Idist': 'origin'}, convert_numerics=True),294    # Build - NHB - TRN - 2 Car295    '912.01': TableDef('912.01', end_table_tag='Total', skip_rows=5,296                       df_drop_top_rows=1, index_col='origin',297                       rename_columns={'Idist': 'origin'}, convert_numerics=True),298    # Build - NHB - TRN - ALL299    '933.01': TableDef('933.01', end_table_tag='Total', skip_rows=5,300                       df_drop_top_rows=1, index_col='origin',301                       rename_columns={'Idist': 'origin'}, convert_numerics=True),302    # Build - All Trips - TRN - ALL303    '954.01': TableDef('954.01', end_table_tag='Total', skip_rows=5,304                       df_drop_top_rows=1, index_col='origin',305                       rename_columns={'Idist': 'origin'}, convert_numerics=True),306    # Build - All Trips - TRN - ALL307    '975.01': TableDef('975.01', end_table_tag='Total', skip_rows=5,308                       df_drop_top_rows=1, index_col='origin',309                       rename_columns={'Idist': 'origin'}, convert_numerics=True),310    # Build - All Trips - TRN - ALL311    '996.01': TableDef('996.01', end_table_tag='Total', skip_rows=5,312                       df_drop_top_rows=1, index_col='origin',313                       rename_columns={'Idist': 'origin'}, convert_numerics=True),314    # Build - All Trips - TRN - ALL315    '1017.01': TableDef('1017.01', end_table_tag='Total', skip_rows=5,316                       df_drop_top_rows=1, index_col='origin',317                       rename_columns={'Idist': 'origin'}, convert_numerics=True),318}319def parse_table(result_file_path, table_label):320    def replace_dash(x):321        return 0 if x == '-' else x322    table_def = _table_parameters[table_label]323    start_table_tag = 'Table{:>9s}\n'.format(table_def.table_id)324    end_table_tag = table_def.end_table_tag325    skip_rows = table_def.skip_rows326    found_table = False327    table = StringIO('')328    with open(result_file_path, 'r') as result_file:329        for line in result_file:330            if line.startswith(start_table_tag):331                found_table = True332            if found_table:333                if line.startswith(end_table_tag):334                    found_table = False335                else:336                    table.write(line)337    table.seek(0)338    df = pd.read_fwf(table, widths=table_def.widths, skiprows=skip_rows)339    if table_def.df_drop_top_rows is not None:340        df = df[table_def.df_drop_top_rows:]341    if table_def.df_drop_tail_rows is not None:342        df = df[:-table_def.df_drop_tail_rows]343    if table_def.reset_header:344        columns = df.columns345        df.columns = np.arange(len(columns))346    if table_def.rename_columns is not None:347        df = df.rename(columns=table_def.rename_columns)348    if table_def.int_columns is not None or table_def.convert_numerics:349        df = df.applymap(np.vectorize(replace_dash))350    if table_def.int_columns is not None:351        df[table_def.int_columns] = df[table_def.int_columns].astype(np.int64)352    if table_def.index_col is not None:353        df = df.set_index(table_def.index_col)354    if table_def.convert_numerics:355        df = df.apply(pd.to_numeric)356    return df.copy()357def summarize_access_modes(result_file_path, percentage=False):358    table_label = '9.01'359    tbl = parse_table(result_file_path, table_label)360    table_def = _table_parameters[table_label]361    362    if not percentage:363        return tbl[table_def.int_columns].sum()364    365    exist = tbl[['exist_wlk','exist_knr', 'exist_pnr', 'exist_xfr']].sum() / tbl[['exist_wlk','exist_knr', 'exist_pnr', 'exist_xfr']].sum().sum()366    nb = tbl[['nb_wlk', 'nb_knr', 'nb_pnr', 'nb_xfr']].sum() / tbl[['nb_wlk', 'nb_knr', 'nb_pnr', 'nb_xfr']].sum().sum()367    bld = tbl[['bld_wlk', 'bld_knr', 'bld_pnr', 'bld_xfr']].sum() / tbl[['bld_wlk', 'bld_knr', 'bld_pnr', 'bld_xfr']].sum().sum()368    369    exist.index = exist.index.str[-3:]370    nb.index = nb.index.str[-3:]371    bld.index = bld.index.str[-3:]372    373    return pd.concat([exist, nb, bld], axis=1, keys=['existing', 'nb', 'bld']).transpose()...

headers_cleanup.py

Source:headers_cleanup.py

1"""2Helper functions to cleanup the headers from the various data sources.3This helps make the final product more human readable.4"""5HEADERS_CHANGE = {6  'elections_2020': {7    'rename_columns': {8      'fips5': 'county_fips'9    },10    'drop_columns': {11      'fips_char', 'place', 'fname', 'lname', 'pab', 'incumbent', 'mpc'12    }13  },14  'census_2019': {15    'rename_columns': {16      'state': 'state_fips_part',17      'county': 'county_fips_part',18      'popestimate2019': 'county_population'19    },20    'drop_columns': {21      'sumlev', 'region', 'division', 'stname', 'ctyname', 'census2010pop',22      'estimatesbase2010', 'popestimate2010', 'popestimate2011',23      'popestimate2012', 'popestimate2013', 'popestimate2014',24      'popestimate2015', 'popestimate2016', 'popestimate2017',25      'popestimate2018', 'npopchg_2010', 'npopchg_2011', 'npopchg_2012',26      'npopchg_2013', 'npopchg_2014', 'npopchg_2015', 'npopchg_2016',27      'npopchg_2017', 'npopchg_2018', 'npopchg_2019', 'births2010',28      'births2011', 'births2012', 'births2013', 'births2014', 'births2015',29      'births2016', 'births2017', 'births2018', 'births2019', 'deaths2010',30      'deaths2011', 'deaths2012', 'deaths2013', 'deaths2014', 'deaths2015',31      'deaths2016', 'deaths2017', 'deaths2018', 'deaths2019', 'naturalinc2010',32      'naturalinc2011', 'naturalinc2012', 'naturalinc2013', 'naturalinc2014',33      'naturalinc2015', 'naturalinc2016', 'naturalinc2017', 'naturalinc2018',34      'naturalinc2019', 'internationalmig2010', 'internationalmig2011',35      'internationalmig2012', 'internationalmig2013', 'internationalmig2014',36      'internationalmig2015', 'internationalmig2016', 'internationalmig2017',37      'internationalmig2018', 'internationalmig2019', 'domesticmig2010',38      'domesticmig2011', 'domesticmig2012', 'domesticmig2013',39      'domesticmig2014', 'domesticmig2015', 'domesticmig2016',40      'domesticmig2017', 'domesticmig2018', 'domesticmig2019', 'netmig2010',41      'netmig2011', 'netmig2012', 'netmig2013', 'netmig2014', 'netmig2015',42      'netmig2016', 'netmig2017', 'netmig2018', 'netmig2019', 'residual2010',43      'residual2011', 'residual2012', 'residual2013', 'residual2014',44      'residual2015', 'residual2016', 'residual2017', 'residual2018',45      'residual2019', 'gqestimatesbase2010', 'gqestimates2010',46      'gqestimates2011', 'gqestimates2012', 'gqestimates2013',47      'gqestimates2014', 'gqestimates2015', 'gqestimates2016',48      'gqestimates2017', 'gqestimates2018', 'gqestimates2019', 'rbirth2011',49      'rbirth2012', 'rbirth2013', 'rbirth2014', 'rbirth2015', 'rbirth2016',50      'rbirth2017', 'rbirth2018', 'rbirth2019', 'rdeath2011', 'rdeath2012',51      'rdeath2013', 'rdeath2014', 'rdeath2015', 'rdeath2016', 'rdeath2017',52      'rdeath2018', 'rdeath2019', 'rnaturalinc2011', 'rnaturalinc2012',53      'rnaturalinc2013', 'rnaturalinc2014', 'rnaturalinc2015',54      'rnaturalinc2016', 'rnaturalinc2017', 'rnaturalinc2018',55      'rnaturalinc2019', 'rinternationalmig2011', 'rinternationalmig2012',56      'rinternationalmig2013', 'rinternationalmig2014', 'rinternationalmig2015',57      'rinternationalmig2016', 'rinternationalmig2017', 'rinternationalmig2018',58      'rinternationalmig2019', 'rdomesticmig2011', 'rdomesticmig2012',59      'rdomesticmig2013', 'rdomesticmig2014', 'rdomesticmig2015',60      'rdomesticmig2016', 'rdomesticmig2017', 'rdomesticmig2018',61      'rdomesticmig2019', 'rnetmig2011', 'rnetmig2012', 'rnetmig2013',62      'rnetmig2014', 'rnetmig2015', 'rnetmig2016', 'rnetmig2017', 'rnetmig2018',63      'rnetmig2019'64    }65  },66  'cdc': {67    'rename_columns': {68      'fips county code': 'county_fips',69      'deaths involving covid-19': 'county_covid19_deaths',70      'deaths from all causes': 'county_all_cause_deaths',71    },72    'drop_columns': [73      'date as of', 'start date', 'end date', 'state', 'county name',74      'urban rural code', 'footnote'75    ]76  },77  'zillow_city_codes': {78    'rename_columns': {79      'code': 'city_code'80    },81    'drop_columns': ['area']82  },83  'experian': {84    'rename_columns': {85      'city ': 'city',86      'vantagescore 3.0 credit score': 'credit score',87      'avg vantagescore 3.0': 'credit score',88      'average vantagescore 3.0 credit score': 'credit score',89      'avg. vantagescore 3.0': 'credit score',90      'weighted vantage score': 'credit score',91      'sum of adjusted credit score': 'credit score',92      ' average vantagescore 3.0 credit score': 'credit score',93      'vantage score': 'credit score',94      'county name': 'county',95    },96    'drop_columns': ['rank', 'population', 'unnamed: 5', 'unnamed: 4']97  },98  'fbi_2015': {99    'rename_columns': {100      'rape (revised definition)1': 'rape',101      'larceny- theft': 'larceny theft',102      'arson3': 'arson',103    },104    'drop_columns': ['rape (legacy definition)2'],105  },106  'fbi_2016': {107    'rename_columns': {108      'rape (revised definition)1': 'rape',109      'larceny- theft': 'larceny theft',110      'arson3': 'arson',111    },112    'drop_columns': ['rape (legacy definition)2'],113  },114  'fbi_2017': {115    'rename_columns': {116      'rape1': 'rape',117      'larceny- theft': 'larceny theft',118      'arson2': 'arson',119    },120    'drop_columns': [121      'unnamed: 13',122      'unnamed: 14',123      'unnamed: 15',124      'unnamed: 16',125      'unnamed: 17',126      'unnamed: 18',127    ]128  },129  'fbi_2018': {130    'rename_columns': {131      'rape1': 'rape',132      'larceny- theft': 'larceny theft',133      'arson2': 'arson',134    },135    'drop_columns': []136  },137  'fbi_2019': {138    'rename_columns': {139      'rape1': 'rape',140      'larceny- theft': 'larceny theft',141      'arson2': 'arson',142    },143    'drop_columns': []144  },145  'census_2010': {146    'rename_columns': {147      'area in square miles - land area': 'land area sqmi census_2010',148      'area in square miles - total area': 'total area sqmi census_2010',149      'area in square miles - water area': 'water area sqmi census_2010',150      'target geo id2': 'geoid'151    },152    'drop_columns': [153      'density per square mile of land area - housing units',154      'density per square mile of land area - population', 'geographic area',155      'geographic area.1', 'geography', 'housing units', 'id', 'id2',156      'population', 'target geo id'157    ]158  },159  'final_csv': {160    'rename_columns': {},161    'drop_columns': [162      'city_code', 'city_fbi_crime', 'city_walkscore', 'cityexperian_2017',163      'cityzillow', 'geography census_2010', 'geography.1', 'geography.2',164      'latitudezillow', 'longitudezillow', 'rape_legacy', 'reverse_address',165      'state_fbi_crime', 'state_fbi_crime', 'state_walkscore',166      'state_walkscore', 'stateexperian_2017', 'statezillow', 'target geo id2',167      'longitude_x', 'longitude_y', 'latitude_x', 'latitude_y',168      'reverse_address_x', 'reverse_address_y'169    ]170  }...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.