How to use log_file method in Robotframework

Best Python code snippet using robotframework

validate

Source:validate

1#!/usr/bin/env python32import os3import pandas as pd4import shutil5import xlrd6import re7import datetime8import time9import Find_Columns10import dbScript11import Add_Column12import traceback13from utilities import get_s3_bucket14from utilities import get_s3_client15from utilities import get_month_from_sys_args16from utilities import get_year_from_sys_args17from utilities import get_month_as_int18from validate_utilities import get_src_filename19from validate_utilities import get_src_file_dir20from validate_utilities import get_src_file_path21from validate_utilities import get_date_from_src_filename22from validate_utilities import get_ext_from_src_filename23from validate_utilities import get_log_path24from validate_utilities import get_year_as_int25from validate_utilities import get_seller_name26from validate_utilities import is_alnum_max_50_chars27from validate_utilities import validate_lead_country_column28from validate_utilities import validate_opportunity_country_column29global Column_Error, file_name_err, Win_Match, start_a2, sellernm_t1, sellernm_t2, sellernm_t3, date, month30def filenmcheck(log_file):31 global Column_missing_tab132 global Column_missing_tab233 global Column_missing_tab334 global Column_Error35 global sellernm_t136 global sellernm_t237 global sellernm_t338 global file_name_err39 file_name_err = 040 Column_Error = 041 sellernm_t1 = 042 sellernm_t2 = 043 sellernm_t3 = 044 timestamp = time.strftime("%H:%M:%S")45 print("*****Validation Report for File {} {}*****\n".format(get_src_filename(), timestamp), file=log_file)46 flag = 047 # download file from S348 try:49 get_s3_client().download_file(get_s3_bucket(),50 get_src_file_path(),51 get_src_file_path())52 except Exception as ex:53 msg_ex = 'Error downloading the file to validate: s3://{}/{} could not be found.'.format(54 get_s3_bucket(),55 get_src_file_path())56 print(msg_ex, file=log_file)57 print(msg_ex)58 raise ex59 wb = xlrd.open_workbook(get_src_file_path())60 sheet = wb.sheet_by_index(0)61 numrows = sheet.nrows 62 sheet_2 = wb.sheet_by_index(1)63 numrows_2 = sheet_2.nrows64 sheet_3 = wb.sheet_by_index(2)65 numrows_3 = sheet_3.nrows66 # --------- CHECKING IF DIRECTORY IS PRESENT ---------67 if not os.path.isdir(get_src_file_dir()):68 print("Please check if the Year and Month is written correctly\nYear:",69 get_year_from_sys_args(),70 "\nMonth:",71 get_month_from_sys_args())72 # all done73 return74 # --------- CHECKING IF FILE IS PRESENT ---------75 if os.path.isfile(get_src_file_path()):76 # --------- FILE NAME AND EXCEL STRUCTURE VALIDATION ---------77 print("\nValidations Part 1 :: File name and Excel structure", file=log_file)78 print("----------------------------------------------------------------", file=log_file)79 print("1. File Name and Excel structure Validation", file=log_file)80 print("\na. File Name Validation", file = log_file)81 print("Filename : {}".format(get_src_filename()), file = log_file)82 print("SellerName : {}".format(get_seller_name()), file = log_file)83 report = get_src_filename().split("_")[0]84 if report != "SellerGTMReport":85 print("start with : {} FAIL".format(report), file = log_file)86 file_name_err += 187 else:88 print("start with : {} PASS".format(report), file = log_file)89 if len(get_date_from_src_filename()) != 8:90 print("Filename Upload Date :{} FAIL".format(get_date_from_src_filename()), file = log_file)91 file_name_err += 192 else:93 print("Filename Upload Date : {} PASS".format(get_date_from_src_filename()), file = log_file)94 ext = get_ext_from_src_filename()95 if ext != "xls" and ext != "xlsx":96 print("File Name Extension : {} FAIL \n".format(ext), file = log_file)97 file_name_err += 198 else:99 print("File Name Extension : {} PASS \n".format(ext), file = log_file)100 print("Total number of rows in tab1 : {}".format(numrows - 1), file=log_file)101 print("Total number of rows in tab2 : {}".format(numrows_2 - 1), file=log_file)102 print("Total number of rows in tab3 : {}".format(numrows_3 - 1), file=log_file)103 print("Total number of rows in tab1 : {}".format(numrows - 1))104 print("Total number of rows in tab2 : {}".format(numrows_2 - 1))105 print("Total number of rows in tab3 : {}".format(numrows_3 - 1))106 # --------- CHECKING IF DATA START FROM A2 ---------107 if (sheet.cell_value(1, 0) == "" or sheet.cell_value(1, 0) == "Seller Company Name*" or sheet.cell_value(1, 0) == "Seller Company Name") or (sheet_3.cell_value(1, 0) == "" or sheet_3.cell_value(1, 0) == "Seller Company Name*" or sheet_3.cell_value(1, 0) == "Seller Company Name"):108 print("Data Start at A2 : FAIL", file = log_file)109 file_name_err += 1110 else:111 print("Data Start at A2 : PASS", file = log_file)112 # --------- ANY COLUMN MISSING VALIDATION ---------113 print("\nb. Excel Structure Validation", file=log_file)114 print("#Excel Structure, Tab 1", file = log_file)115 Column_missing_tab1, misplaced_column_1 = Find_Columns.find_columns_tab1(get_src_file_path(), log_file)116 print("\n#Excel Structure, Tab 2", file = log_file)117 Column_missing_tab2, misplaced_column_2 = Find_Columns.find_columns_tab2(get_src_file_path(), log_file)118 print("\n#Excel Structure, Tab 3", file = log_file)119 Column_missing_tab3, misplaced_column_3 = Find_Columns.find_columns_tab3(get_src_file_path(), log_file)120 if Column_missing_tab1 or Column_missing_tab2 or Column_missing_tab3:121 Column_Error = len(Column_missing_tab1) + len(Column_missing_tab2) + len(Column_missing_tab3) + len(misplaced_column_1) + len(misplaced_column_2) + len(misplaced_column_3) 122 else:123 print("No such file:", get_src_file_path())124 # all done125 return126 print("\n", file=log_file) 127 if flag == 0:128 # --------- CHECKING IF SELLER NAME PRESENT IN DATABASE ---------129 print("\n2. Seller Company Name Validation (Checks the existance of sellername from the File name in the Database table : mst_SellerParticipants)", file = log_file)130 seller_id = dbScript.get_seller_id(get_seller_name())131 if seller_id != 0:132 print("Seller Name from table - mst_SellerParticipants : {} PASS (Seller Company Name exists.)".format(get_seller_name()), file = log_file)133 print("Seller UID from table - mst_SellerParticipants : {} PASS".format(seller_id), file=log_file)134 else:135 print("Seller Name from table - mst_SellerParticipants : {} FAIL (Seller Company Name does not exist.)".format(get_seller_name()), file = log_file)136 file_name_err += 1137 print("\n########################################################", file=log_file)138 tab1reader(log_file)139# --------- TAB 1 VALIDATIONS ---------140def tab1reader(log_file):141 global Column_missing_tab1142 global Column_missing_tab2143 global Column_missing_tab3144 global sellernm_t1145 global sellernm_t2146 global sellernm_t3147 global Column_Error148 global file_name_err149 wb = xlrd.open_workbook(get_src_file_path())150 sheet = wb.sheet_by_index(0)151 numrows = sheet.nrows152 numcols = sheet.ncols153 regex = re.compile('[@_!#$%^&*()<>?/\|}{~:]')154 xlsx = pd.ExcelFile(get_src_file_path())155 df = pd.read_excel(get_src_file_path())156 Seller_Company_Name_error = 0157 GTM_Campaign_Source = 0158 Campaign_Name = 0159 CRM_System_Campaign_ID = 0160 Campaign_Create_Date = 0161 Lead_ID = 0162 Create_Date = 0163 err_cnt_lead_country = 0164 Lead_Status = 0 # Changes are done (Valid,junk ..etc)165 print("\nValidations Part 2 and Part 3 for data validation :: Column Validations Tab 1", file=log_file)166 print("----------------------------------------------------------------", file=log_file)167 print("\n3. Column Validations for Tab 1 for incorrect size and incorrect data", file=log_file)168 for i in range(0, numcols):169 one_based_column_position = i + 1170 col_name = re.sub('[*]', '', sheet.cell_value(0, i))171 if col_name == "Seller Company Name":172 print("\nSeller Company Name Column::", file=log_file)173 for y in range(1, numrows):174 seller_company_name = sheet.cell_value(y, i)175 try:176 if type(seller_company_name) == str and regex.search(seller_company_name) == None \177 and seller_company_name != '' and len(seller_company_name) <= 50:178 seller_company_name179 else:180 Seller_Company_Name_error += 1181 print("Seller Company Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", seller_company_name, file=log_file)182 except:183 Seller_Company_Name_error += 1184 print("Seller Company Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", seller_company_name, file=log_file)185 passed = numrows - Seller_Company_Name_error186 if Seller_Company_Name_error == 0 and one_based_column_position not in Column_missing_tab1:187 print("Validation on Seller Company Name : PASS (Satisfied Conditions :: Alpha, 50 Characters)", file = log_file)188 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)189 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error), file = log_file)190 else:191 print("Validation on Seller Company Name : FAIL", file = log_file)192 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)193 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error), file = log_file)194 elif col_name == "GTM Campaign Source":195 print("\nGTM Campaign Source Column::", file=log_file)196 for y in range(1, numrows):197 gtm_campaign_source = sheet.cell_value(y, i)198 try:199 if (gtm_campaign_source.lower() == 'aws-led' or gtm_campaign_source.lower()== 'seller-led' or gtm_campaign_source.lower() == 'joint') and (gtm_campaign_source != ''):200 gtm_campaign_source201 else:202 GTM_Campaign_Source += 1203 print("GTM Campaign Source Error on", "Row:", y + 1, "Column:", i + 1, "Value:",204 gtm_campaign_source, file=log_file)205 except:206 GTM_Campaign_Source += 1207 print("GTM Campaign Source Error on", "Row:", y + 1, "Column:", i + 1, "Value:", gtm_campaign_source, file=log_file)208 passed = numrows - GTM_Campaign_Source209 if GTM_Campaign_Source == 0 and one_based_column_position not in Column_missing_tab1:210 print("Validation on GTM Campaign Source : PASS (Satisfied Conditions ::Must be either AWS-led, Seller-led, or Joint)", file = log_file)211 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)212 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source), file = log_file)213 else:214 print("Validation on GTM Campaign Source : FAIL", file = log_file)215 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)216 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source), file = log_file)217 elif col_name == "Campaign Name":218 print("\nCampaign Name Column::", file=log_file)219 for y in range(1, numrows):220 campaign_name = sheet.cell_value(y, i)221 try:222 if type(campaign_name) == str and len(campaign_name) <= 100 and campaign_name != '':223 campaign_name224 else:225 Campaign_Name += 1226 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_name,227 file=log_file)228 except:229 Campaign_Name += 1230 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_name, file=log_file)231 passed = numrows - Campaign_Name232 if Campaign_Name == 0 and one_based_column_position not in Column_missing_tab1:233 print("Validation on Campaign Name : PASS (Satisfied Conditions :: Alpha, 80 Characters)", file = log_file)234 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)235 print("Total number of rows FAILED : {}".format(Campaign_Name), file = log_file)236 else:237 print("Validation on Campaign Name : FAIL", file = log_file)238 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)239 print("Total number of rows FAILED : {}".format(Campaign_Name), file = log_file)240 elif col_name == "CRM System Campaign ID":241 print("\nCRM System Campaign Column::", file=log_file)242 for y in range(1, numrows):243 crm_system_campaign_id = sheet.cell_value(y, i)244 try:245 if not is_alnum_max_50_chars(crm_system_campaign_id):246 CRM_System_Campaign_ID += 1247 print("CRM System Campaign ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:",248 crm_system_campaign_id, file=log_file)249 except:250 CRM_System_Campaign_ID += 1251 print("CRM System Campaign ID Exception on", "Row:", y + 1, "Column:", i + 1, "Value:", crm_system_campaign_id, file=log_file)252 passed = numrows - CRM_System_Campaign_ID253 if CRM_System_Campaign_ID == 0 and one_based_column_position not in Column_missing_tab1:254 print("Validation on CRM System Campaign ID : PASS (Satisfied Conditions :: Alpha, 30 Characters)", file = log_file)255 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)256 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID), file = log_file)257 else:258 print("Validation on CRM System Campaign ID : FAIL", file = log_file)259 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)260 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID), file = log_file)261 elif col_name == "Campaign Create Date":262 print("\nCampaign Create Date Column::", file=log_file)263 for y in range(1, numrows): # for traversing rows264 campaign_create_date = sheet.cell_value(y, i)265 try:266 if str(campaign_create_date) != '':267 date = str(campaign_create_date)268 length = len(date)269 real_date = date[:length]270 if len(real_date) == 10:271 month = int(date[:2])272 day = int(date[2:4])273 if 1 <= month <= 12:274 if 1 <= day <= 31:275 continue276 else:277 Campaign_Create_Date += 1278 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)279 else:280 Campaign_Create_Date += 1281 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)282 elif len(real_date) == 9:283 month = int(date[:1])284 day = int(date[1:3])285 if 1 <= month <= 12:286 if 1 <= day <= 31:287 continue288 else:289 Campaign_Create_Date += 1290 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)291 else:292 Campaign_Create_Date += 1293 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)294 elif len(real_date) <= 7:295 try:296 date = str(xlrd.xldate.xldate_as_datetime(campaign_create_date, wb.datemode))297 year, month, tmix = date.split('-')298 day, timevar = tmix.split(' ')299 isValidDate = True300 datetime.datetime(int(year), int(month), int(day))301 except:302 isValidDate = False303 if (isValidDate):304 continue305 else:306 Campaign_Create_Date += 1307 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)308 else:309 Campaign_Create_Date += 1310 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)311 else:312 Campaign_Create_Date += 1313 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)314 except:315 Campaign_Create_Date += 1316 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)317 passed = numrows - Campaign_Create_Date318 if Campaign_Create_Date == 0 and one_based_column_position not in Column_missing_tab1:319 print("Validation on Campaign Create Date : PASS (Satisfied Conditions :: Alpha, Data format mmddyyyy)", file = log_file)320 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)321 print("Total number of rows FAILED : {}".format(Campaign_Create_Date), file = log_file)322 else:323 print("Validation on Campaign Create Date : FAIL", file = log_file)324 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)325 print("Total number of rows FAILED : {}".format(Campaign_Create_Date), file = log_file)326 elif col_name == "Lead ID":327 print("\nLead ID Column::", file=log_file)328 for y in range(1, numrows):329 lead_id = sheet.cell_value(y, i)330 try:331 if not is_alnum_max_50_chars(lead_id):332 Lead_ID += 1333 print("Lead ID Error on", "Row:", y + 1, "Column:", i + 1, lead_id, file=log_file)334 except:335 Lead_ID += 1336 print("Lead ID Error on", "Row:", y + 1, "Column:", i + 1, lead_id, file=log_file)337 passed = numrows - Lead_ID338 if Lead_ID == 0 and one_based_column_position not in Column_missing_tab1:339 print("Validation on Lead ID : PASS (Satisfied Conditions :: Alpha, 30 Characters)", file = log_file)340 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)341 print("Total number of rows FAILED : {}".format(Lead_ID), file = log_file)342 else:343 print("Validation on Lead ID : FAIL", file = log_file)344 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)345 print("Total number of rows FAILED : {}".format(Lead_ID), file = log_file)346 elif col_name == "Create Date":347 print("\nCreate Date Column::", file=log_file)348 for y in range(1, numrows):349 create_date = sheet.cell_value(y, i)350 try:351 if str(create_date) != '':352 date = str(create_date)353 length = len(date)354 real_date = date[:length]355 if len(real_date) == 10:356 month = int(date[:2])357 day = int(date[2:4])358 if 1 <= month <= 12:359 if 1 <= day <= 31:360 continue361 else:362 Create_Date += 1363 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)364 else:365 Create_Date += 1366 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)367 elif len(real_date) == 9:368 month = int(date[:1])369 day = int(date[1:3])370 if 1 <= month <= 12:371 if 1 <= day <= 31:372 continue373 else:374 Create_Date += 1375 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)376 else:377 Create_Date += 1378 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)379 elif len(real_date) <= 7:380 try:381 date = str(xlrd.xldate.xldate_as_datetime(create_date, wb.datemode))382 year, month, tmix = date.split('-')383 day, timevar = tmix.split(' ')384 isValidDate = True385 datetime.datetime(int(year), int(month), int(day))386 except:387 isValidDate = False388 if (isValidDate):389 continue390 else:391 Create_Date += 1392 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)393 else:394 Create_Date += 1395 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)396 else:397 Create_Date += 1398 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)399 except:400 Create_Date += 1401 print("Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", create_date, file=log_file)402 passed = numrows - Create_Date403 if Create_Date == 0 and one_based_column_position not in Column_missing_tab1:404 print("Validation on Create Date : PASS (Satisfied Conditions :: Alpha, Data format mmddyyyy)", file = log_file)405 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)406 print("Total number of rows FAILED : {}".format(Create_Date), file = log_file)407 else:408 print("Validation on Create Date : FAIL", file = log_file)409 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)410 print("Total number of rows FAILED : {}".format(Create_Date), file = log_file)411 elif col_name == 'Lead Country':412 print('\nLead Country Column::', file=log_file)413 err_cnt_lead_country = validate_lead_country_column(sheet, numrows, i, Column_missing_tab1, one_based_column_position, log_file)414 elif col_name == "Lead Status": # newly added Lead Status415 print("\nLead Status Column::", file=log_file)416 for y in range(1, numrows):417 lead_status = sheet.cell_value(y, i)418 try:419 if (lead_status.lower() == 'valid' or lead_status.lower() == 'closed' or lead_status.lower() == 'lost' or lead_status.lower() == 'junk') \420 and str(lead_status != ''):421 lead_status422 else:423 Lead_Status += 1424 print("Lead Status Error on", "Row:", y + 1, "Column:", i + 1, "Value:", lead_status, file=log_file)425 except:426 Lead_Status += 1427 print("Lead Status Error on", "Row:", y + 1, "Column:", i + 1, "Value:", lead_status, file=log_file)428 passed = numrows - Lead_Status429 if Lead_Status == 0 and one_based_column_position not in Column_missing_tab1:430 print("Validation on Lead Status : PASS (Satisfied Conditions :: Must be either Valid, Closed, Lost, or Junk)", file = log_file)431 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)432 print("Total number of rows FAILED : {}".format(Lead_Status), file = log_file)433 else:434 print("Validation on Lead Status : FAIL", file = log_file)435 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)436 print("Total number of rows FAILED : {}".format(Lead_Status), file = log_file)437 ERRTab1List = [Seller_Company_Name_error, GTM_Campaign_Source, Campaign_Name, CRM_System_Campaign_ID, Campaign_Create_Date, Lead_ID,438 Create_Date, err_cnt_lead_country, Lead_Status, sellernm_t1, sellernm_t2, sellernm_t3, Column_Error, file_name_err] 439 tab2reader(ERRTab1List, log_file, df, sheet, xlsx) 440def tab2reader(ERRTab1List, log_file, df, sheet, xlsx):441 wb = xlrd.open_workbook(get_src_file_path())442 sheet1 = wb.sheet_by_index(1)443 numrows = sheet1.nrows444 numcols = sheet1.ncols445 regex = re.compile('[@_!#$%^&*()<>?/\|}{~:]')446 global Column_missing_tab1447 global Column_missing_tab2448 global Column_missing_tab3449 global sellernm_t1450 global sellernm_t2451 global sellernm_t3452 global Column_Error453 global file_name_err454 Seller_Company_Name_error = 0455 GTM_Campaign_Source = 0456 Campaign_Name = 0457 CRM_System_Campaign_ID = 0458 Campaign_Create_Date = 0459 Opportunity_ID = 0460 Convert_Date = 0461 err_cnt_opportunity_country = 0462 Opportunity_Status = 0 # newly adjusted(Valid,junk..etc) 463 AWS_Marketplace_Opportunity = 0464 Pipeline_Revenue = 0465 Account_Id = 0466 Account_Name_error = 0467 Win_Loss = 0468 Billed_Revenue = 0469 print("\nValidations Part 2 and Part 3 for data validation :: Column Validations Tab 2", file=log_file)470 print("----------------------------------------------------------------", file=log_file)471 print("\n3. Column Validations for Tab 2 for incorrect size and incorrect data", file=log_file)472 for i in range(0, numcols):473 one_based_column_position = i + 1474 col_name = re.sub('[*]', '', sheet1.cell_value(0, i)).replace('\n', ' ')475 if col_name == "Seller Company Name":476 print("\nSeller Company Name Column::", file=log_file)477 for y in range(1, numrows):478 seller_company_name = sheet1.cell_value(y, i)479 try:480 if type(seller_company_name) == str and regex.search(seller_company_name) == None \481 and seller_company_name != '' and len(seller_company_name) <= 50:482 seller_company_name483 else:484 Seller_Company_Name_error += 1485 print("Seller Company Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", seller_company_name, file=log_file)486 except:487 Seller_Company_Name_error += 1488 print("Seller Company Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", seller_company_name, file=log_file)489 passed = numrows - Seller_Company_Name_error490 if Seller_Company_Name_error == 0 and one_based_column_position not in Column_missing_tab2:491 print("Validation on Seller Company Name : PASS (Satisfied Conditions :: Alpha, 50 Characters)", file = log_file)492 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)493 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error), file = log_file)494 else:495 print("Validation on Seller Company Name : FAIL", file = log_file)496 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)497 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error), file = log_file)498 elif col_name == "GTM Campaign Source":499 print("\nGTM Campaign Source Column::", file=log_file)500 for y in range(1, numrows):501 gtm_campaign_source = sheet1.cell_value(y, i)502 try:503 if (gtm_campaign_source.lower() == 'aws-led' or gtm_campaign_source.lower()== 'seller-led' or gtm_campaign_source.lower() == 'joint') and (gtm_campaign_source != ''):504 gtm_campaign_source505 else:506 GTM_Campaign_Source += 1507 print("GTM Campaign Source Error on", "Row:", y + 1, "Column:", i + 1, "Value:",508 gtm_campaign_source, file=log_file)509 except:510 GTM_Campaign_Source += 1511 print("GTM Campaign Source Error on", "Row:", y + 1, "Column:", i + 1, "Value:", gtm_campaign_source, file=log_file)512 passed = numrows - GTM_Campaign_Source513 if GTM_Campaign_Source == 0 and one_based_column_position not in Column_missing_tab2:514 print("Validation on GTM Campaign Source : PASS (Satisfied Conditions ::Must be either AWS-led, Seller-led, or Joint)", file = log_file)515 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)516 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source), file = log_file)517 else:518 print("Validation on GTM Campaign Source : FAIL", file = log_file)519 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)520 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source), file = log_file)521 elif col_name == "Campaign Name":522 print("\nCampaign Name Column::", file=log_file)523 for y in range(1, numrows):524 campaign_name = sheet1.cell_value(y, i)525 try:526 if type(campaign_name) == str and len(campaign_name) <= 100 and campaign_name != '':527 campaign_name528 else:529 Campaign_Name += 1530 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_name,531 file=log_file)532 except:533 Campaign_Name += 1534 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_name, file=log_file)535 passed = numrows - Campaign_Name536 if Campaign_Name == 0 and one_based_column_position not in Column_missing_tab2:537 print("Validation on Campaign Name : PASS (Satisfied Conditions :: Alpha, 80 Characters)", file = log_file)538 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)539 print("Total number of rows FAILED : {}".format(Campaign_Name), file = log_file)540 else:541 print("Validation on Campaign Name : FAIL", file = log_file)542 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)543 print("Total number of rows FAILED : {}".format(Campaign_Name), file = log_file)544 elif col_name == "CRM System Campaign ID":545 print("\nCRM System Campaign Column::", file=log_file)546 for y in range(1, numrows):547 crm_system_campaign_id = sheet1.cell_value(y, i)548 try:549 if not is_alnum_max_50_chars(crm_system_campaign_id):550 CRM_System_Campaign_ID += 1551 print("CRM System Campaign ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:",552 crm_system_campaign_id, file=log_file)553 except:554 CRM_System_Campaign_ID += 1555 print("CRM System Campaign ID Exception on", "Row:", y + 1, "Column:", i + 1, "Value:", crm_system_campaign_id, file=log_file)556 passed = numrows - CRM_System_Campaign_ID557 if CRM_System_Campaign_ID == 0 and one_based_column_position not in Column_missing_tab2:558 print("Validation on CRM System Campaign ID : PASS (Satisfied Conditions :: Alpha, 30 Characters)", file = log_file)559 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)560 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID), file = log_file)561 else:562 print("Validation on CRM System Campaign ID : FAIL", file = log_file)563 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)564 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID), file = log_file)565 elif col_name == "Campaign Create Date":566 print("\nCampaign Create Date Column::", file=log_file)567 for y in range(1, numrows): # for traversing rows568 campaign_create_date = sheet1.cell_value(y, i)569 try:570 if str(campaign_create_date) != '':571 date = str(campaign_create_date)572 length = len(date)573 real_date = date[:length]574 if len(real_date) == 10:575 month = int(date[:2])576 day = int(date[2:4])577 if 1 <= month <= 12:578 if 1 <= day <= 31:579 continue580 else:581 Campaign_Create_Date += 1582 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)583 else:584 Campaign_Create_Date += 1585 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)586 elif len(real_date) == 9:587 month = int(date[:1])588 day = int(date[1:3])589 if 1 <= month <= 12:590 if 1 <= day <= 31:591 continue592 else:593 Campaign_Create_Date += 1594 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)595 else:596 Campaign_Create_Date += 1597 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)598 elif len(real_date) <= 7:599 try:600 date = str(xlrd.xldate.xldate_as_datetime(campaign_create_date, wb.datemode))601 year, month, tmix = date.split('-')602 day, timevar = tmix.split(' ')603 isValidDate = True604 datetime.datetime(int(year), int(month), int(day))605 except:606 isValidDate = False607 if (isValidDate):608 continue609 else:610 Campaign_Create_Date += 1611 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)612 else:613 Campaign_Create_Date += 1614 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)615 else:616 Campaign_Create_Date += 1617 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)618 except:619 Campaign_Create_Date += 1620 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", campaign_create_date, file=log_file)621 passed = numrows - Campaign_Create_Date622 if Campaign_Create_Date == 0 and one_based_column_position not in Column_missing_tab2:623 print("Validation on Campaign Create Date : PASS (Satisfied Conditions :: Alpha, Data format mmddyyyy)", file = log_file)624 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)625 print("Total number of rows FAILED : {}".format(Campaign_Create_Date), file = log_file)626 else:627 print("Validation on Campaign Create Date : FAIL", file = log_file)628 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)629 print("Total number of rows FAILED : {}".format(Campaign_Create_Date), file = log_file)630 elif col_name == "Opportunity ID": # newly adjusted (Was at column 11)631 print("\nOpportunity ID Column::", file=log_file)632 for y in range(1, numrows):633 opportunity_id = sheet1.cell_value(y, i)634 try:635 if str(opportunity_id) != '' and len(str(opportunity_id)) > 30:636 Opportunity_ID += 1637 print("Opportunity ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:", opportunity_id,638 file=log_file)639 except:640 Opportunity_ID += 1641 print("Opportunity ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:", opportunity_id, file=log_file)642 passed = numrows - Opportunity_ID643 if Opportunity_ID == 0 and one_based_column_position not in Column_missing_tab2:644 print("Validation on Opportunity ID : PASS (Satisfied Conditions :: Alpha, 30 Characters)", file = log_file)645 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)646 print("Total number of rows FAILED : {}".format(Opportunity_ID), file = log_file)647 else:648 print("Validation on Opportunity ID : FAIL", file = log_file)649 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)650 print("Total number of rows FAILED : {}".format(Opportunity_ID), file = log_file)651 elif col_name == "Convert Date":652 print("\nConvert Date Column::", file=log_file)653 for y in range(1, numrows):654 convert_date = sheet1.cell_value(y, i)655 try:656 if str(convert_date) != '':657 date = str(convert_date)658 length = len(date)659 real_date = date[:length]660 if len(real_date) == 10:661 month = int(date[:2])662 day = int(date[2:4])663 if 1 <= month <= 12:664 if 1 <= day <= 31:665 continue666 else:667 Convert_Date += 1668 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)669 else:670 Convert_Date += 1671 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)672 elif len(real_date) == 9:673 month = int(date[:1])674 day = int(date[1:3])675 if 1 <= month <= 12:676 if 1 <= day <= 31:677 continue678 else:679 Convert_Date += 1680 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)681 else:682 Convert_Date += 1683 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)684 elif len(real_date) <= 7:685 try:686 date = str(xlrd.xldate.xldate_as_datetime(convert_date, wb.datemode))687 year, month, tmix = date.split('-')688 day, timevar = tmix.split(' ')689 isValidDate = True690 datetime.datetime(int(year), int(month), int(day))691 except:692 isValidDate = False693 if (isValidDate):694 continue695 else:696 Convert_Date += 1697 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)698 else:699 Convert_Date += 1700 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)701 else:702 continue703 except:704 Convert_Date += 1705 print("Convert Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", convert_date, file=log_file)706 passed = numrows - Convert_Date707 if Convert_Date == 0 and one_based_column_position not in Column_missing_tab2:708 print("Validation on Convert Date : PASS (Satisfied Conditions :: Data format mmddyyyy)", file = log_file)709 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)710 print("Total number of rows FAILED : {}".format(Convert_Date), file = log_file)711 else:712 print("Validation on Convert Date : FAIL", file = log_file)713 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)714 print("Total number of rows FAILED : {}".format(Convert_Date), file = log_file)715 elif col_name == 'Opportunity Country':716 print('\nOpportunity Country Column::', file=log_file)717 err_cnt_opportunity_country = validate_opportunity_country_column(sheet1, numrows, i, Column_missing_tab2, one_based_column_position, log_file)718 elif col_name == "Opportunity Status": # newly added Opportunity Status719 print("\nOpportunity Status Column::", file=log_file)720 for y in range(1, numrows):721 opportunity_status = sheet1.cell_value(y, i)722 try:723 if opportunity_status.lower() == 'valid' or opportunity_status.lower() == 'closed' or opportunity_status.lower() == \724 'lost' or opportunity_status.lower() == 'junk' or str(opportunity_status) == '':725 opportunity_status726 else:727 Opportunity_Status += 1728 print("Opportunity Status Error on", "Row:", y + 1, "Column:", i + 1, "Value:", opportunity_status,729 file=log_file)730 except:731 Opportunity_Status += 1732 print("Opportunity Status Error on", "Row:", y + 1, "Column:", i + 1, "Value:", opportunity_status, file=log_file)733 passed = numrows - Opportunity_Status734 if Opportunity_Status == 0 and one_based_column_position not in Column_missing_tab2:735 print("Validation on Opportunity Status : PASS (Satisfied Conditions :: Must be either Valid, Closed, Lost, or Junk)", file = log_file)736 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)737 print("Total number of rows FAILED : {}".format(Opportunity_Status), file = log_file)738 else:739 print("Validation on Opportunity Status : FAIL", file = log_file)740 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)741 print("Total number of rows FAILED : {}".format(Opportunity_Status), file = log_file)742 elif col_name == "AWS Marketplace Opportunity": # newly added previously AWS_Marketplace_Opportunity?743 print("\nAWS Marketplace Opportunity Column::", file=log_file)744 for y in range(1, numrows):745 aws_marketplace_opportunity = sheet1.cell_value(y, i)746 try:747 if aws_marketplace_opportunity.lower() == 'yes' or aws_marketplace_opportunity.lower() == 'no' or aws_marketplace_opportunity.lower() == 'not reportable' or aws_marketplace_opportunity == '':748 aws_marketplace_opportunity749 else:750 AWS_Marketplace_Opportunity += 1751 print("AWS Marketplace Opportunity Error on", "Row:", y + 1, "Column:", i + 1, "Value:",752 aws_marketplace_opportunity, file=log_file)753 except:754 AWS_Marketplace_Opportunity += 1755 print("AWS Marketplace Opportunity Error on", "Row:", y + 1, "Column:", i + 1, "Value:", aws_marketplace_opportunity, file=log_file)756 passed = numrows - AWS_Marketplace_Opportunity757 if AWS_Marketplace_Opportunity == 0 and one_based_column_position not in Column_missing_tab2:758 print("Validation on AWS Marketplace Opportunity : PASS (Satisfied Conditions :: Must be either Yes, No, or Not Reportable)", file = log_file)759 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)760 print("Total number of rows FAILED : {}".format(AWS_Marketplace_Opportunity), file = log_file)761 else:762 print("Validation on AWS Marketplace Opportunity : FAIL", file = log_file)763 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)764 print("Total number of rows FAILED : {}".format(AWS_Marketplace_Opportunity), file = log_file)765 elif col_name == "Pipeline Revenue":766 # in document its mentioned that Numeric, comma separated, 11 characters767 # but no commas are given in .xl file768 print("\nPipeline Revenue Column::", file=log_file)769 minus_values = 0770 zero_values = []771 for y in range(1, numrows): # for traversing rows772 pipeline_revenue = sheet1.cell_value(y, i)773 try:774 if pipeline_revenue != '':775 data = int(pipeline_revenue)776 datastr = str(data)777 if (type(data) == int) and int(data) >= 0 or (str(pipeline_revenue) == '') and (len(datastr) <= 11):778 if data == 0:779 zero_values.append(str(y+1))780 else:781 Pipeline_Revenue += 1782 print("Pipeline Revenue Error on", "Row:", y + 1, "Column:", i + 1, "Value:",783 pipeline_revenue, file=log_file)784 except:785 Pipeline_Revenue += 1786 print("Pipeline Revenue Error on", "Row:", y + 1, "Column:", i + 1, "Value:", pipeline_revenue, file=log_file)787 passed = numrows - Pipeline_Revenue788 if Pipeline_Revenue == 0 and one_based_column_position not in Column_missing_tab2:789 print("Validation on Pipeline Revenue : PASS (Satisfied Conditions :: Alpha, 11 Characters)", file = log_file)790 else:791 print("Validation on Pipeline Revenue : FAIL", file = log_file)792 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)793 if len(zero_values) != 0:794 print("Total number of rows FLAGGED : {} (Row {})".format(len(zero_values), ', '.join(zero_values)), file = log_file)795 print("Total number of rows FAILED : {}".format(Pipeline_Revenue), file = log_file)796 elif col_name == "Account Name":797 print("\nAccount Name Column::", file=log_file)798 for y in range(1, numrows):799 account_name = sheet1.cell_value(y, i)800 if account_name in [None, '']:801 continue802 try:803 if type(account_name) == str:804 account_name805 else:806 Account_Name_error += 1807 print("Account Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", account_name, file=log_file)808 except:809 Account_Name_error += 1 810 print("Account Name Error on", "Row:", y + 1, "Column:", i + 1, "Value:", account_name, file=log_file)811 passed = numrows - Account_Name_error812 if Account_Name_error == 0 and one_based_column_position not in Column_missing_tab2:813 print("Validation on Account Name : PASS (Satisfied Conditions :: Alpha, 50 Characters)", file = log_file)814 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)815 print("Total number of rows FAILED : {}".format(Account_Name_error), file = log_file)816 else:817 print("Validation on Account Name : FAIL", file = log_file)818 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)819 print("Total number of rows FAILED : {}".format(Account_Name_error), file = log_file)820 elif col_name == "Account ID":821 # in document its mentioned that Numeric, comma separated, 11 characters822 # but no commas are given in .xl file823 print("\nAccount ID Column::", file=log_file)824 for y in range(1, numrows): # for traversing rows825 account_id = sheet1.cell_value(y, i)826 if account_id in [None, '']:827 continue828 try: 829 datastr = str(account_id)830 if len(datastr) <= 50:831 account_id832 else:833 Account_Id += 1834 print("Account ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:",835 account_id, file=log_file)836 except:837 Account_Id += 1838 print("Account ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:", account_id, file=log_file)839 passed = numrows - Account_Id840 if Account_Id == 0 and one_based_column_position not in Column_missing_tab2:841 print("Validation on Account ID : PASS (Satisfied Conditions :: Alpha, 11 Characters)", file = log_file)842 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)843 print("Total number of rows FAILED : {}".format(Account_Id), file = log_file)844 else:845 print("Validation on Account ID : FAIL", file = log_file)846 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)847 print("Total number of rows FAILED : {}".format(Account_Id), file = log_file)848 elif col_name == "Win Date":849 print("\nWin Date Column::", file=log_file)850 for y in range(1, numrows): # for traversing rows851 win_date = sheet1.cell_value(y, i)852 try:853 if str(win_date) != '':854 date = xlrd.xldate.xldate_as_datetime(win_date, wb.datemode)855 date = str(date)856 year, month, tmix = date.split('-')857 day, timevar = tmix.split(' ')858 isValidDate = True859 try:860 datetime.datetime(int(year), int(month), int(day))861 except ValueError:862 isValidDate = False863 if not isValidDate:864 Win_Loss += 1865 print("Win Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", win_date,866 file=log_file)867 else:868 continue869 except:870 Win_Loss += 1871 print("Win Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", win_date, file=log_file)872 passed = numrows - Win_Loss873 if Win_Loss == 0 and one_based_column_position not in Column_missing_tab2:874 print("Validation on Win Date : PASS (Satisfied Conditions :: Data format mmddyyyy)", file = log_file)875 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)876 print("Total number of rows FAILED : {}".format(Win_Loss), file = log_file)877 else:878 print("Validation on Win Date : FAIL", file = log_file)879 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)880 print("Total number of rows FAILED : {}".format(Win_Loss), file = log_file)881 elif col_name == "Billed Revenue":882 print("\nBilled Revenue Column::", file=log_file)883 minus_values = 0884 single_values = []885 for y in range(1, numrows): # for traversing rows886 billed_revenue = sheet1.cell_value(y, i)887 try:888 if billed_revenue != '':889 data = int(billed_revenue)890 datastr = str(data)891 if (type(data) == int) and int(data) >= 0 or (str(billed_revenue) == '') and (len(datastr) <= 11):892 if len(str(data)) == 1:893 single_values.append(str(y+1))894 else:895 Billed_Revenue += 1896 print("Billed Revenue Error on", "Row:", y + 1, "Column:", i + 1, "Value:", billed_revenue,897 file=log_file)898 else:899 continue900 except:901 Billed_Revenue += 1902 print("Billed Revenue Error on", "Row:", y + 1, "Column:", i + 1, "Value:", billed_revenue, file=log_file)903 passed = numrows - Billed_Revenue904 if Billed_Revenue == 0 and one_based_column_position not in Column_missing_tab2:905 print("Validation on Billed Revenue : PASS (Satisfied Conditions :: Alpha, 11 Characters)", file = log_file)906 else:907 print("Validation on Billed Revenue : FAIL", file = log_file)908 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)909 if len(single_values) != 0:910 print("Total number of rows FLAGGED : {} (Row {})".format(len(single_values), ', '.join(single_values)), file = log_file)911 print("Total number of rows FAILED : {}".format(Billed_Revenue), file = log_file)912 ERRTab2List = [Seller_Company_Name_error, GTM_Campaign_Source, Campaign_Name, CRM_System_Campaign_ID,913 Campaign_Create_Date, Opportunity_ID, Convert_Date, err_cnt_opportunity_country, 914 Opportunity_Status, AWS_Marketplace_Opportunity, Pipeline_Revenue, Account_Name_error, 915 Account_Id, Win_Loss, Billed_Revenue]916 tab3reader(ERRTab1List, ERRTab2List, log_file, df, sheet, sheet1, xlsx)917 log_file.close()918# --------- RULE 2 VALIDATIONS ---------919def Rule_3Check(ERRTab1List, ERRTab2List, ERRTab3List, log_file, df, sheet, sheet1, xlsx):920 global date921 global month922 CampName_CRMId_err = 0923 lead_camp_Err = 0924 oppor_id_Err = 0925 valid_opp_Err = 0926 Win_Match = 0927 Duplicampaign_1 = 0928 Duplicampaign_2 = 0929 Duplicampaign_3 = 0930 Duplicampaign = 0931 numrows = sheet.nrows932 numrows1 = sheet1.nrows933 global Column_missing_tab1934 global Column_missing_tab2935 global Column_missing_tab3936 print("\nValidations Part 3 :: Additional validations", file=log_file)937 print("----------------------------------------------------------------", file=log_file)938 '''Validate Campaign Name / ID pairs in .xls file'''939 940 # CHECKING OPPORTUNITY COLUMNS MISSING OR NOT941 pass_values_opportunity = 0942 fail_values_opportunity = 0943 print("\n6. Validate Conditionally Required Fields :: 2 scenarios", file = log_file)944 print("Validate conditional opportunity fields", file=log_file)945 for y in range(1, numrows1):946 try:947 if sheet1.cell_value(y, 5) == '':948 if sheet1.cell_value(y, 6) == '' and sheet1.cell_value(y, 7) == "" and sheet1.cell_value(y, 8) == '' and sheet1.cell_value(y, 9) == '' and sheet1.cell_value(y, 10) == '':949 pass_values_opportunity += 1950 else:951 valid_opp_Err += 1952 fail_values_opportunity += 1953 print("Values found at:", y + 1, sheet1.cell_value(y, 6), sheet1.cell_value(y, 7),954 sheet1.cell_value(y, 8)955 , sheet1.cell_value(y, 9), sheet1.cell_value(y, 10), "Should be blank instead", file=log_file)956 else:957 if sheet1.cell_value(y, 6) != '' and sheet1.cell_value(y, 7) != '' and sheet1.cell_value(y, 8) != '' and sheet1.cell_value(y, 9) != '' and sheet1.cell_value(y, 10) != '':958 pass_values_opportunity += 1959 else:960 valid_opp_Err += 1961 fail_values_opportunity += 1962 if sheet1.cell_value(y, 6) == '':963 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 6), " FAIL", file=log_file)964 if sheet1.cell_value(y, 7) == '':965 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 7), " FAIL", file=log_file)966 if sheet1.cell_value(y, 8) == '':967 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 8), " FAIL", file=log_file)968 if sheet1.cell_value(y, 9) == '':969 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 9), " FAIL", file=log_file)970 if sheet1.cell_value(y, 10) == '':971 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 10), " FAIL", file=log_file)972 except Exception as ex:973 fail_values_opportunity += 1974 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 11), " FAIL", file=log_file)975 if valid_opp_Err == 0:976 print("Conditional Opportunity fields : PASS", file=log_file)977 print("TOTAL ROWS PASS : {}".format(pass_values_opportunity), file=log_file)978 print("TOTAL ROWS FAIL : {}".format(fail_values_opportunity), file=log_file)979 else:980 print("Conditional Opportunity fields : FAIL", file=log_file)981 print("TOTAL ROWS PASS : {}".format(pass_values_opportunity), file=log_file)982 print("TOTAL ROWS FAIL : {}".format(fail_values_opportunity), file=log_file)983 pass_values_win = 0984 fail_values_win = 0985 print("\nValidate conditional win date fields", file=log_file)986 for y in range(1, numrows1):987 try:988 if sheet1.cell_value(y, 13) == "":989 if sheet1.cell_value(y, 14) == "":990 pass_values_win += 1991 else:992 Win_Match += 1993 fail_values_win += 1994 print("Values found at:", y + 1, sheet1.cell_value(y, 14), "Should be blank instead", file=log_file)995 else:996 if sheet1.cell_value(y, 13) != "":997 if sheet1.cell_value(y, 14) != "":998 pass_values_win += 1999 else:1000 Win_Match += 11001 fail_values_win += 11002 if sheet1.cell_value(y, 14) != '':1003 print("Values found at row:", y + 1, "is/are blank at:", sheet1.cell_value(0, 14), " FAIL", file=log_file)1004 except:1005 fail_values_win += 11006 print("Values found at:", y + 1, sheet1.cell_value(y, 14), "Something's wrong", file=log_file)1007 if Win_Match == 0:1008 print("Conditional win date fields : PASS", file=log_file)1009 print("TOTAL ROWS PASS : {}".format(pass_values_win), file=log_file)1010 print("TOTAL ROWS FAIL : {}".format(fail_values_win), file=log_file)1011 else:1012 print("Conditional win date fields : FAIL", file=log_file)1013 print("TOTAL ROWS PASS : {}".format(pass_values_win), file=log_file)1014 print("TOTAL ROWS FAIL : {}".format(fail_values_win), file=log_file)1015 shit_names = xlsx.sheet_names1016 print("\n8. Validate unique Campaign Name and Campaign ID pairs", file=log_file)1017 print("Campaign names and Campaign IDs match within tab 1", file=log_file)1018 try:1019 colname_1 = df.columns[2]1020 campname = df[colname_1].unique()1021 colname_3 = df.columns[3]1022 crmid = df[colname_3].unique()1023 pass_values_camp = 01024 fail_values_camp = 01025 if len(crmid) == len(campname):1026 for uniqval in range(0, len(crmid)):1027 for y in range(1, numrows):1028 if campname[uniqval] == sheet.cell_value(y, 2):1029 if crmid[uniqval] == sheet.cell_value(y, 3):1030 pass_values_camp += 11031 else:1032 CampName_CRMId_err += 11033 fail_values_camp += 11034 print("Not matching Row:", y + 1, sheet.cell_value(y, 2), sheet.cell_value(y, 3), file=log_file)1035 if CampName_CRMId_err == 0:1036 print("Unique opportunities within a campaign : PASS", file=log_file)1037 print("TOTAL ROWS PASS : {}".format(pass_values_camp), file=log_file)1038 print("TOTAL ROWS FAIL : {}".format(fail_values_camp), file=log_file)1039 else:1040 print("Unique opportunities within a campaign : PASS", file=log_file)1041 print("TOTAL ROWS PASS : {}".format(pass_values_camp), file=log_file)1042 print("TOTAL ROWS FAIL : {}".format(fail_values_camp), file=log_file)1043 else:1044 CampName_CRMId_err += 11045 fail_values_camp += 11046 print("Number of Campaign Names are not eqaul to number of Unique Campaign IDs\nNumber of Campaign Names:",1047 len(campname),1048 "\nNumber of Campaign IDs:", len(crmid), "\nCampaign Names:", campname, "\nCampaign IDs:", crmid,1049 file=log_file)1050 except:1051 CampName_CRMId_err += 11052 print("Campaign Name and ID's Validation : FAIL (Maybe data does not start from A2)", file=log_file)1053 print("\nCompare Tab 1, Tab 2 and Tab 3 to ensure that Campaign Name and Campaign ID pairs match", file = log_file)1054 try:1055 df1 = pd.read_excel(xlsx, shit_names[0])1056 pos = 21057 colname_1 = df1.columns[pos]1058 pos = 31059 colname_3 = df1.columns[pos] 1060 uniqueObj = pd.DataFrame(df1, columns=[colname_1, colname_3])1061 uniqueValues = uniqueObj.drop_duplicates()1062 list_Unique_1 = uniqueValues.values.tolist()1063 for index in range(0, len(list_Unique_1)):1064 for index_1 in range(index + 1, len(list_Unique_1)):1065 if index + 1 != len(list_Unique_1):1066 if list_Unique_1[index][0] == list_Unique_1[index_1][0] or list_Unique_1[index][1] == list_Unique_1[index_1][1]:1067 Duplicampaign_1 += 11068 print("Duplicates found::", list_Unique_1[index], list_Unique_1[index_1], file=log_file)1069 except:1070 Duplicampaign_1 += 11071 print("Campaign Name and ID's Validation for Tab1 : FAIL (Maybe data does not start from A2)", file=log_file)1072 if Duplicampaign_1 != 0:1073 print("Validate number of campaign name and campaign ID pairs in Tab 1 : FAIL", file=log_file)1074 else:1075 print("Validate number of campaign name and campaign ID pairs in Tab 1 : PASS", file=log_file)1076 try:1077 df2 = pd.read_excel(xlsx, shit_names[1])1078 pos = 21079 colname_1 = df2.columns[pos]1080 pos = 31081 colname_3 = df2.columns[pos]1082 uniqueObj = pd.DataFrame(df2, columns=[colname_1, colname_3])1083 uniqueValues = uniqueObj.drop_duplicates()1084 list_Unique_2 = uniqueValues.values.tolist()1085 for index in range(0, len(list_Unique_2)):1086 for index_1 in range(index + 1, len(list_Unique_2)):1087 if index + 1 != len(list_Unique_2):1088 if list_Unique_2[index][0] == list_Unique_2[index_1][0] or list_Unique_2[index][1] == list_Unique_2[index_1][1]:1089 Duplicampaign_2 += 11090 print("Duplicates found::", list_Unique_2[index], list_Unique_2[index_1], file=log_file)1091 except:1092 Duplicampaign_2 += 11093 print("Campaign Name and ID's Validation for Tab2 : FAIL (Maybe data does not start from A2)", file=log_file)1094 if Duplicampaign_2 != 0:1095 print("Validate number of campaign name and campaign ID pairs in Tab 2 : FAIL", file=log_file)1096 else:1097 print("Validate number of campaign name and campaign ID pairs in Tab 2 : PASS", file=log_file)1098 try:1099 df3 = pd.read_excel(xlsx, shit_names[2])1100 pos = 21101 colname_1 = df3.columns[pos]1102 pos = 31103 colname_3 = df3.columns[pos]1104 uniqueObj = pd.DataFrame(df3, columns=[colname_1, colname_3])1105 uniqueValues = uniqueObj.drop_duplicates()1106 list_Unique_3 = uniqueValues.values.tolist()1107 for index in range(0, len(list_Unique_3)):1108 for index_1 in range(index + 1, len(list_Unique_3)):1109 if index + 1 != len(list_Unique_3):1110 if list_Unique_3[index][0] == list_Unique_3[index_1][0] or list_Unique_3[index][1] == list_Unique_3[index_1][1]:1111 Duplicampaign_3 += 11112 print("Duplicates found::", list_Unique_3[index], list_Unique_3[index_1], file=log_file)1113 except:1114 Duplicampaign_3 += 11115 print("Campaign Name and ID's Validation for Tab3 : FAIL (Maybe data does not start from A2)", file=log_file)1116 if Duplicampaign_3 != 0:1117 print("Validate number of campaign name and campaign ID pairs in Tab 3 : FAIL", file=log_file)1118 else:1119 print("Validate number of campaign name and campaign ID pairs in Tab 3 : PASS", file=log_file)1120 list_Unique_mix = list_Unique_11121 for unique in list_Unique_2:1122 if unique not in list_Unique_mix:1123 list_Unique_mix.append(unique)1124 if len(list_Unique_mix) == len(list_Unique_3):1125 set_x = set([i[0] for i in list_Unique_mix])1126 set_z = set([i[0] for i in list_Unique_3])1127 if set_x == set_z:1128 print("Validate number of campaign name and campaign ID pairs between Tab 1 + Tab 2 and Tab 3 are consistent : PASS", file=log_file)1129 else:1130 Duplicampaign += 11131 print("Validate number of campaign name and campaign ID pairs between Tab 1 + Tab 2 and Tab 3 are consistent : FAIL", file=log_file)1132 print("Tab1 + Tab2 List :", file=log_file)1133 for index in range(0, len(list_Unique_mix)):1134 print(list_Unique_mix[index], file=log_file) 1135 print("Tab3 List :", file=log_file)1136 for index in range(0, len(list_Unique_3)):1137 print(list_Unique_3[index], file=log_file)1138 else:1139 Duplicampaign += 11140 print("Validate number of campaign name and campaign ID pairs between Tab 1 + Tab 2 and Tab 3 are consistent : FAIL", file=log_file)1141 print("Tab1 + Tab2 List :", file=log_file)1142 for index in range(0, len(list_Unique_mix)):1143 print(list_Unique_mix[index], file=log_file)1144 print("\nTab3 List :", file=log_file)1145 for index in range(0, len(list_Unique_3)):1146 print(list_Unique_3[index], file=log_file)1147 1148 seller_uid = dbScript.get_seller_id(get_seller_name())1149 prev_lead, prev_opportunity, prev_win_loss, prev_campaign, prev_tot_investment = dbScript.fetch_counts(get_year_from_sys_args(), get_month_as_int(), seller_uid)1150 curr_lead, curr_opportunity, curr_win_loss, curr_campaign, curr_tot_investment = Find_Columns.get_current_count(get_src_file_path())1151 flag = 0 1152 print("\n##################################################################", file=log_file)1153 print("\nValidations Part 4 :: Validation with Counter Table", file=log_file)1154 print("--------------------------------------------------------------------", file=log_file)1155 print("\n10. Previous Vs Current Counts/Totals", file=log_file)1156 if curr_lead >= prev_lead:1157 print("Previous Lead Count : {} | Current Lead Count : {} | PASS".format(prev_lead, curr_lead), file=log_file)1158 else:1159 flag += 11160 print("Previous Lead Count : {} | Current Lead Count : {} | FAIL".format(prev_lead, curr_lead), file=log_file)1161 if curr_opportunity >= prev_opportunity:1162 print("Previous Opportunity Count : {} | Current Opportunity Count : {} | PASS".format(prev_opportunity, curr_opportunity), file=log_file)1163 else:1164 flag += 11165 print("Previous Opportunity Count : {} | Current Opportunity Count : {} | FAIL".format(prev_opportunity, curr_opportunity), file=log_file)1166 if curr_win_loss >= prev_win_loss:1167 print("Previous Win Date Count : {} | Current Win Date Count : {} | PASS".format(prev_win_loss, curr_win_loss), file=log_file)1168 else:1169 flag += 11170 print("Previous Win Date Count : {} | Current Win Date Count : {} | FAIL".format(prev_win_loss, curr_win_loss), file=log_file)1171 if curr_campaign >= prev_campaign:1172 print("Previous Campaign Count : {} | Current Campaign Count : {} | PASS".format(prev_campaign, curr_campaign), file=log_file)1173 else:1174 flag += 11175 print("Previous Campaign Count : {} | Current Campaign Count : {} | FAIL".format(prev_campaign, curr_campaign), file=log_file)1176 if curr_tot_investment >= prev_tot_investment:1177 print("Previous Total Investment : {} | Current Total Investment : {} | PASS".format(prev_tot_investment, curr_tot_investment), file=log_file)1178 else:1179 flag += 11180 print("Previous Total Investment : {} | Current Total Investment : {} | FAIL".format(prev_tot_investment, curr_tot_investment), file=log_file)1181 if flag == 0:1182 counterval=01183 else:1184 counterval=11185 reqparam=[counterval, curr_lead, curr_opportunity, curr_win_loss, curr_campaign, curr_tot_investment]1186 1187 final_R2Count = [CampName_CRMId_err, lead_camp_Err, oppor_id_Err, valid_opp_Err, Win_Match, flag, Duplicampaign, Duplicampaign_1, Duplicampaign_2, Duplicampaign_3]1188 print("\n################################################################", file=log_file) 1189 count_error(get_src_filename(), ERRTab1List, ERRTab2List, ERRTab3List, final_R2Count, log_file, reqparam)1190# --------- TAB 2 VALIDATIONS ---------1191def tab3reader(ERRTab1List, ERRTab2List, log_file, df, sheet, sheet1, xlsx):1192 wb = xlrd.open_workbook(get_src_file_path())1193 sheet2 = wb.sheet_by_index(2)1194 numrows = sheet2.nrows1195 numcols = sheet2.ncols1196 regex = re.compile('[@_!#$%^&*()<>?/\|}{~:]')1197 Seller_Company_Name_error_tab3 = 01198 GTM_Campaign_Source_tab3 = 01199 Campaign_Name_tab3 = 01200 CRM_System_Campaign_ID_tab3 = 01201 Campaign_Create_Date_tab3 = 01202 Investment_error_tab3 = 01203 print("\n4. Column Validations for Tab 3", file=log_file)1204 for i in range(0, numcols): # for traversing Columns1205 col_name = re.sub('[*]', '', sheet2.cell_value(0, i))1206 if col_name == "Seller Company Name":1207 print("Seller Company Name Column::", file=log_file)1208 for y in range(1, numrows):1209 try:1210 if type(sheet2.cell_value(y, i)) == str and regex.search(sheet2.cell_value(y, i)) == None \1211 and sheet2.cell_value(y, i) != '' and len(sheet2.cell_value(y, i)) <= 50:1212 (sheet2.cell_value(y, i))1213 else:1214 Seller_Company_Name_error_tab3 += 11215 print("Seller_Company_Name Error on", "Row:", y + 1, "Column:", i, "Value:",1216 sheet2.cell_value(y, i), file=log_file)1217 except:1218 Seller_Company_Name_error_tab3 += 11219 print("Seller_Company_Name Error on", "Row:", y + 1, "Column:", i, "Value:", sheet2.cell_value(y, i), file=log_file)1220 passed = numrows - Seller_Company_Name_error_tab31221 if Seller_Company_Name_error_tab3 == 0:1222 print("Seller Company Name Validation : PASS (Satisfied Conditions :: Alpha, 50 Characters)", file = log_file)1223 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1224 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error_tab3), file = log_file)1225 else:1226 print("Seller Company Name Validation : FAIL", file = log_file)1227 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1228 print("Total number of rows FAILED : {}".format(Seller_Company_Name_error_tab3), file = log_file)1229 elif col_name == "GTM Campaign Source":1230 print("\nGTM Campaign Source Column::", file=log_file)1231 for y in range(1, numrows):1232 try:1233 value=sheet2.cell_value(y, i)1234 if (value.lower() == 'aws-led' or value.lower()== 'seller-led' or value.lower() == 'joint') and (sheet2.cell_value(y, i) != ''):1235 sheet2.cell_value(y, i)1236 else:1237 GTM_Campaign_Source_tab3 += 11238 print("GTM_Campaign_Source Error on", "Row:", y + 1, "Column:", i, "Value:",1239 sheet2.cell_value(y, i), file=log_file)1240 except:1241 GTM_Campaign_Source_tab3 += 11242 print("GTM_Campaign_Source Error on", "Row:", y + 1, "Column:", i, "Value:", sheet2.cell_value(y, i), file=log_file)1243 passed = numrows - GTM_Campaign_Source_tab31244 if GTM_Campaign_Source_tab3 == 0:1245 print("GTM Campaign Source Validation : PASS (Satisfied Conditions :: Must be either AWS-led, Seller-led, or Joint)", file = log_file)1246 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1247 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source_tab3), file = log_file)1248 else:1249 print("GTM Campaign Source Validation : FAIL", file = log_file)1250 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1251 print("Total number of rows FAILED : {}".format(GTM_Campaign_Source_tab3), file = log_file)1252 1253 elif col_name == "Campaign Name":1254 print("\nCampaign Name Column::", file=log_file)1255 for y in range(1, numrows):1256 try:1257 if type(sheet2.cell_value(y, i)) == str and len(sheet2.cell_value(y, i)) <= 100 and sheet2.cell_value(y, i) != '':1258 sheet2.cell_value(y, i)1259 else:1260 Campaign_Name_tab3 += 11261 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i, "Value:", sheet2.cell_value(y, i),1262 file=log_file)1263 except:1264 Campaign_Name_tab3 += 11265 print("Campaign_Name Error on", "Row:", y + 1, "Column:", i, "Value:", sheet2.cell_value(y, i),1266 file=log_file)1267 passed = numrows - Campaign_Name_tab31268 if Campaign_Name_tab3 == 0:1269 print("Campaign Name Validation : PASS (Satisfied Conditions :: Alpha, 80 Characters)", file = log_file)1270 print("Total number of rows PASSED : {}".format(passed - 1), file=log_file)1271 print("Total number of rows FAILED : {}".format(Campaign_Name_tab3), file=log_file)1272 else:1273 print("Campaign Name Validation : FAIL", file = log_file)1274 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1275 print("Total number of rows FAILED : {}".format(Campaign_Name_tab3), file = log_file)1276 elif col_name == "CRM System Campaign ID":1277 print("\nCRM System Campaign ID Column::", file=log_file)1278 for y in range(1, numrows):1279 try:1280 if not is_alnum_max_50_chars(sheet.cell_value(y, i)):1281 CRM_System_Campaign_ID_tab3 += 11282 print("CRM System Campaign ID Error on", "Row:", y + 1, "Column:", i + 1, "Value:",1283 sheet.cell_value(y, i), file=log_file)1284 except:1285 CRM_System_Campaign_ID_tab3 += 11286 print("CRM System Campaign ID Exception on", "Row:", y + 1, "Column:", i, "Value:", sheet2.cell_value(y, i), file=log_file)1287 passed = numrows - CRM_System_Campaign_ID_tab31288 if CRM_System_Campaign_ID_tab3 == 0:1289 print("CRM System Campaign ID Validation : PASS (Satisfied Conditions :: Alpha, 30 Characters)", file = log_file)1290 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1291 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID_tab3), file = log_file)1292 else:1293 print("CRM System Campaign ID Validation : FAIL", file = log_file)1294 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1295 print("Total number of rows FAILED : {}".format(CRM_System_Campaign_ID_tab3), file = log_file)1296 elif col_name == "Campaign Create Date":1297 print("\nCampaign Create Date Column::", file=log_file)1298 for y in range(1, numrows):1299 try:1300 if str(sheet2.cell_value(y, i)) != '':1301 date = str(sheet2.cell_value(y, i))1302 length = len(date)1303 real_date = date[:length]1304 if len(real_date) == 10:1305 month = int(date[:2])1306 day = int(date[2:4])1307 if 1 <= month <= 12:1308 if 1 <= day <= 31:1309 continue1310 else:1311 Campaign_Create_Date_tab3 += 11312 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1313 else:1314 Campaign_Create_Date_tab3 += 11315 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1316 elif len(real_date) == 9:1317 month = int(date[:1])1318 day = int(date[1:3])1319 if 1 <= month <= 12:1320 if 1 <= day <= 31:1321 continue1322 else:1323 Campaign_Create_Date_tab3 += 11324 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1325 else:1326 Campaign_Create_Date_tab3 += 11327 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1328 elif len(real_date) <= 7:1329 try:1330 date = str(xlrd.xldate.xldate_as_datetime(sheet2.cell_value(y, i), wb.datemode))1331 year, month, tmix = date.split('-')1332 day, timevar = tmix.split(' ')1333 isValidDate = True1334 datetime.datetime(int(year), int(month), int(day))1335 except:1336 isValidDate = False1337 if (isValidDate):1338 continue1339 else:1340 Campaign_Create_Date_tab3 += 11341 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1342 else:1343 Campaign_Create_Date_tab3 += 11344 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1345 else:1346 Campaign_Create_Date_tab3 += 11347 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1348 except:1349 Campaign_Create_Date_tab3 += 11350 print("Campaign Create Date Error on", "Row:", y + 1, "Column:", i + 1, "Value:", sheet2.cell_value(y, i), file=log_file)1351 passed = numrows - Campaign_Create_Date_tab31352 if Campaign_Create_Date_tab3 == 0:1353 print("Validation on Campaign Create Date : PASS (Satisfied Conditions :: Alpha, Data format mmddyyyy)", file = log_file)1354 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1355 print("Total number of rows FAILED : {}".format(Campaign_Create_Date_tab3), file = log_file)1356 else:1357 print("Validation on Campaign Create Date : FAIL", file = log_file)1358 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1359 print("Total number of rows FAILED : {}".format(Campaign_Create_Date_tab3), file = log_file)1360 elif col_name == "Investment":1361 print("\nInvestment Column::", file=log_file)1362 for y in range(1, numrows):1363 try:1364 if sheet2.cell_value(y, i) is not '':1365 str2 = sheet2.cell_value(y, i)1366 rege = re.compile('$')1367 check_str = str(str2)1368 if rege.search(check_str) is not None:1369 check_str = check_str.replace("$", "")1370 str2 = float(check_str)1371 regex = re.compile('[@_!#%^&*()<>?/\|}{~:]')1372 if(regex.search(str(str2)) == None): 1373 data = int(float(str2))1374 datastr = str(data)1375 if (type(data) == int) and (len(datastr) <= 11) and datastr != '':1376 sheet2.cell_value(y, i)1377 else:1378 Investment_error_tab3 += 11379 print("Investment Error", "Row:", y + 1, "Column:", i, "Values", sheet2.cell_value(y, i), file=log_file)1380 else: 1381 Investment_error_tab3 += 11382 print("Investment Error", "Row:", y + 1, "Column:", i, "Values", sheet2.cell_value(y, i), file=log_file)1383 else:1384 Investment_error_tab3 += 11385 print("Investment Error", "Row:", y + 1, "Column:", i, "Values", sheet2.cell_value(y, i), file=log_file)1386 except Exception as msg:1387 print("In Exception......")1388 print(msg)1389 Investment_error_tab3 += 11390 print("Investment Error", "Row:", y + 1, "Column:", i, "Values", sheet2.cell_value(y, i), file=log_file)1391 passed = numrows - Investment_error_tab31392 if Investment_error_tab3 == 0:1393 print("Investment Error Validation : PASS (Satisfied Conditions :: Alpha, 11 Characters)", file = log_file)1394 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1395 print("Total number of rows FAILED : {}".format(Investment_error_tab3), file = log_file)1396 else:1397 print("Investment Error Validation : FAIL", file = log_file)1398 print("Total number of rows PASSED : {}".format(passed - 1), file = log_file)1399 print("Total number of rows FAILED : {}".format(Investment_error_tab3), file = log_file)1400 ERRTab3List = [Seller_Company_Name_error_tab3, GTM_Campaign_Source_tab3, Campaign_Name_tab3,1401 CRM_System_Campaign_ID_tab3, Campaign_Create_Date_tab3, Investment_error_tab3] 1402 print("\n################################################################", file=log_file)1403 Rule_3Check(ERRTab1List, ERRTab2List, ERRTab3List, log_file, df, sheet, sheet1, xlsx)1404# --------- COUNT OF TOTAL ERRORS ---------1405def count_error(filename, errlisttypo, errlistTab2, errlistTab3, errlistr2, log_file,reqparam):1406 if sum(errlisttypo + errlistr2 + errlistTab2 + errlistTab3) != 0:1407 print("\n+++++Tab1 Error+++++", file=log_file)1408 print("\nSeller Company Name Error : ", errlisttypo[0], file=log_file)1409 print("GTM Campaign Source Error : ", errlisttypo[1], file=log_file)1410 print("Campaign Name Error : ", errlisttypo[2], file=log_file)1411 print("CRM System Campaign ID Error : ", errlisttypo[3], file=log_file)1412 print("Campaign Create Date Error : ", errlisttypo[4], file=log_file)1413 print("Lead ID Error : ", errlisttypo[5], file=log_file)1414 print("Create Date Error : ", errlisttypo[6], file=log_file)1415 print("Lead Country Error : ", errlisttypo[7], file=log_file) # newly added1416 print("Lead Status Error : ", errlisttypo[8], file=log_file)1417 print("\n+++++Tab2 Error+++++", file=log_file)1418 print("\nSeller Company Name Error : ", errlistTab2[0], file=log_file)1419 print("GTM Campaign Source Error : ", errlistTab2[1], file=log_file)1420 print("Campaign Name Error : ", errlistTab2[2], file=log_file)1421 print("CRM System Campaign ID Error : ", errlistTab2[3], file=log_file)1422 print("Campaign Create Date Error : ", errlistTab2[4], file=log_file)1423 print("Opportunity ID Error : ", errlistTab2[5], file=log_file)1424 print("Convert Date Error : ", errlistTab2[6], file=log_file)1425 print("Opportunity Country Error : ", errlistTab2[7], file=log_file)1426 print("Opportunity Status Error : ", errlistTab2[8], file=log_file) # newly added Opportunity_Status1427 print("AWS Marketplace Opportunity Error : ", errlistTab2[9], file=log_file)1428 print("Pipeline Revenue Error : ", errlistTab2[10], file=log_file)1429 print("Account Name : ", errlistTab2[11], file=log_file)1430 print("Account ID : ", errlistTab2[12], file=log_file)1431 print("Win Date Error : ", errlistTab2[13], file=log_file)1432 print("Billed Revenue Error : ", errlistTab2[14], file=log_file)1433 print("\n+++++Tab3 Errors+++++", file=log_file)1434 print("Seller Company Name_error : ", errlistTab3[0], file=log_file)1435 print("GTM Campaign Source Error : ", errlistTab3[1], file=log_file)1436 print("Campaign Name Error : ", errlistTab3[2], file=log_file)1437 print("CRM System Campaign ID Error : ", errlistTab3[3], file=log_file)1438 print("Campaign Create Date Error : ", errlistTab3[4], file=log_file)1439 print("Investment Error : ", errlistTab3[5], file=log_file)1440 r1count = sum(errlisttypo)1441 r2count = sum(errlistTab2)1442 r3count = sum(errlistTab3)1443 r4count = sum(errlistr2) 1444 totalcount = r1count + r2count + r3count + r4count 1445 print("\nTotal Errors in Tab1, Tab2 and Tab3 : ", totalcount, file=log_file)1446 if totalcount != 0:1447 print("\n%s has not been validated and saved to _Rejected and renamed %s_REJECTED.xls. "1448 "FAIL" % (filename, filename), file=log_file)1449 print("\n************************************END OF FILE************************************", file=log_file)1450 filerejected()1451 elif totalcount == 0:1452 fileaccepted(reqparam, log_file)1453 close_and_upload_log(log_file)1454def close_and_upload_log(log_file):1455 log_file.close()1456 # upload log to S31457 get_s3_client().upload_file(log_file.name, get_s3_bucket(), log_file.name)1458 print('Log file has been uploaded to {}.'.format(log_file.name))1459def filerejected():1460 dbScript.update_file_data(get_year_as_int(),1461 get_month_as_int(),1462 get_seller_name(),1463 get_src_filename(),1464 "Fail")1465 print("File:",get_src_file_path(),"is Rejected! Please check Logs in Log Files for more information.")1466 fail_loc = '{}/{}/Rejected Files/{}__REJECTED.{}'.format(1467 get_year_from_sys_args(),1468 get_month_from_sys_args(),1469 get_src_filename(),1470 get_ext_from_src_filename()1471 )1472 shutil.copy(get_src_file_path(), fail_loc)1473 # upload fail.xlsx1474 get_s3_client().upload_file(fail_loc,1475 get_s3_bucket(),1476 fail_loc)1477def fileaccepted(reqparam, log_file):1478 counterval=reqparam[0]1479 curr_lead=reqparam[1]1480 curr_opportunity=reqparam[2]1481 curr_win_loss=reqparam[3]1482 curr_campaign=reqparam[4]1483 curr_tot_investment=reqparam[5]1484 # move the file to the accepted location1485 success_loc = '{}/{}/Successful Files/{}__SUCCESS.{}'.format(1486 get_year_from_sys_args(),1487 get_month_from_sys_args(),1488 get_src_filename(),1489 get_ext_from_src_filename()1490 )1491 shutil.copy(get_src_file_path(), success_loc)1492 # upload success.xlsx1493 get_s3_client().upload_file(success_loc,1494 get_s3_bucket(),1495 success_loc)1496 # GET COPIED FILE LOCATION BY CREATING COPY1497 seller_uid = dbScript.get_seller_id(get_seller_name())1498 copy_loc = 'temp_for_tabs_prep.xlsx'1499 Add_Column.create_copy(get_src_file_path(), copy_loc)1500 Add_Column.create_column(seller_uid, copy_loc, get_year_from_sys_args(), get_month_as_int())1501 xlsx = pd.ExcelFile(copy_loc)1502 wb_copy = xlrd.open_workbook(copy_loc)1503 os.remove(copy_loc)1504 sheet_copied = wb_copy.sheet_names()1505 # allowing values for Lead Country that are 'N/A', which is None by default, so using keep_default_na=False1506 df1 = pd.read_excel(xlsx, sheet_copied[1], keep_default_na=False)1507 df2 = pd.read_excel(xlsx, sheet_copied[2])1508 df3 = pd.read_excel(xlsx, sheet_copied[3])1509 colname_1 = df1.columns[2]1510 colname_2 = df2.columns[2]1511 colname_3 = df3.columns[2]1512 df1[colname_1] = df1[colname_1].str.replace(',', '')1513 df1[colname_1] = df1[colname_1].str.replace('\'', r'\'')1514 df1[colname_1] = df1[colname_1].str.replace('\"', r'\"')1515 df1[colname_2] = df1[colname_2].str.replace(',', '')1516 df1[colname_2] = df1[colname_2].str.replace('\'', r'\'')1517 df1[colname_2] = df1[colname_2].str.replace('\"', r'\"')1518 df3[colname_3] = df3[colname_3].str.replace(',', '')1519 df3[colname_3] = df3[colname_3].str.replace('\'', r'\'')1520 df3[colname_3] = df3[colname_3].str.replace('\"', r'\"')1521 colname_1 = df1.columns[1]1522 df1[colname_1] = df1[colname_1].str.lower()1523 df3[colname_1] = df3[colname_1].str.lower()1524 # write tab1 to csv1525 tab1_filename = '{}Tab1_accepted.csv'.format(get_src_filename())1526 tab1_filepath = '{}/{}/Successful Files/{}'.format(1527 get_year_from_sys_args(),1528 get_month_from_sys_args(),1529 tab1_filename1530 )1531 df1.to_csv(tab1_filepath,1532 sep=',',1533 header=False,1534 index=False,1535 date_format='%m/%d/%Y',1536 float_format="%.0f")1537 # upload tab1 to S31538 get_s3_client().upload_file(tab1_filepath, get_s3_bucket(), tab1_filepath)1539 # write tab2 to csv1540 tab2_filename = '{}Tab2_accepted.csv'.format(get_src_filename())1541 tab2_filepath = '{}/{}/Successful Files/{}'.format(1542 get_year_from_sys_args(),1543 get_month_from_sys_args(),1544 tab2_filename1545 )1546 df2.to_csv(tab2_filepath,1547 sep=',',1548 header=False,1549 index=False,1550 date_format='%m/%d/%Y',1551 float_format="%.0f")1552 # upload tab1 to S31553 get_s3_client().upload_file(tab2_filepath, get_s3_bucket(), tab2_filepath)1554 # write tab3 to csv1555 tab3_filename = '{}Tab3_accepted.csv'.format(get_src_filename())1556 tab3_filepath = '{}/{}/Successful Files/{}'.format(1557 get_year_from_sys_args(),1558 get_month_from_sys_args(),1559 tab3_filename1560 )1561 df3.to_csv(tab3_filepath,1562 sep=',',1563 header=False,1564 index=False,1565 date_format='%m/%d/%Y',1566 float_format="%.0f")1567 # upload tab3 to S31568 get_s3_client().upload_file(tab3_filepath,1569 get_s3_bucket(),1570 tab3_filepath)1571 #Update the counter table in redshift1572 if counterval==0:1573 dbScript.verify_master_data(get_year_from_sys_args(),1574 get_month_as_int(),1575 seller_uid,1576 get_seller_name())1577 dbScript.insert_count(get_year_from_sys_args(),1578 get_month_as_int(),1579 seller_uid,1580 get_seller_name(),1581 curr_lead,1582 curr_opportunity,1583 curr_win_loss,1584 curr_campaign,1585 int(curr_tot_investment))1586 print("\n{} has been validated and saved to _Success and renamed {}_SUCCESS.xls. "1587 "SUCCESS".format(get_src_filename(), get_src_filename()), file=log_file)1588 print('File: {} is Accepted! and Excel sheet and csv files are added to Successful Files Folder.'.format(1589 get_src_file_path()))1590 dbScript.update_file_data(get_year_from_sys_args(), get_month_as_int(), get_seller_name(), get_src_filename(), "Pass")1591# START OF CODE EXECUTION1592log_file = None1593try:1594 log_file = open(get_log_path(), "w")1595 print('Starting validation for {}'.format(get_src_file_path()))1596 Column_missing_tab1 = []1597 Column_missing_tab2 = []1598 Column_missing_tab3 = []1599 filenmcheck(log_file)1600 print('End of validation for {}'.format(get_src_file_path()))1601except Exception:1602 msg_exception = '!!! ERROR RUNNING VALIDATION: AN EXCEPTION WAS RAISED:'1603 if log_file is not None:1604 print(msg_exception, file=log_file)1605 traceback.print_exc(file=log_file)1606 close_and_upload_log(log_file)1607 print(msg_exception)...

pipeline.py

Source:pipeline.py

1""" Basic pipeline for the computation of the recommendation scores.2"""3from subprocess import run4import subprocess5from time import time6rounds = "10"7distance = "4"8path = ""9with open("experiment_parameters.txt", "w") as file_one:10 file_one.write(f"rounds: {rounds}\n")11 file_one.write(f"embedding distance: {distance}\n")12with open("log.txt", "w") as log_file:13 log_file.write("script,time\n")14 start = time()15 # 0. Get PageRank.16 print("Get PageRank")17 run(["cp", path + "CppFiles/getPagerank.out", "."])18 cp = run(19 ["./getPagerank.out"],20 universal_newlines=True,21 stdout=subprocess.PIPE,22 stderr=subprocess.PIPE,23 )24 log_file.write(f"getPagerank.out{time() - start}\n")25 log_file.write(f"{cp.stdout}\n")26 log_file.write(f"{cp.stderr}\n")27 log_file.write(f"{cp.returncode}\n")28 temp_start = time()29 # 1. Get source nodes.30 print("Get source nodes")31 run(["cp", path + "PythonFiles/getSourceNodes.py", "."])32 cp = run(33 [34 "python3",35 "getSourceNodes.py",36 "-p",37 "random",38 "-a",39 rounds,40 "-o",41 "random_source_nodes.csv",42 ],43 universal_newlines=True,44 stdout=subprocess.PIPE,45 stderr=subprocess.PIPE,46 )47 # cp = run(["python3", "getSourceNodes.py", "-p", "best-pagerank", "-a", rounds, "-o", "best_by_pagerank_nodes.csv"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)48 # cp = run(["python3", "getSourceNodes.py", "-p", "worst-pagerank", "-a", rounds, "-o", "worst_by_pagerank_nodes.csv"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)49 log_file.write(f"getSourceNodes.py{time() - temp_start}\n")50 log_file.write(f"{cp.stdout}\n")51 log_file.write(f"{cp.stderr}\n")52 log_file.write(f"{cp.returncode}\n")53 temp_start = time()54 # 2. Get candidate edges.55 print("Get candidate edges")56 run(["cp", path + "PythonFiles/getCandidateEdges.py", "."])57 cp = run(58 [59 "python3",60 "getCandidateEdges.py",61 "-i",62 "random_source_nodes.csv",63 "-d",64 distance,65 "-o",66 "candidate_edges.csv",67 ],68 universal_newlines=True,69 stdout=subprocess.PIPE,70 stderr=subprocess.PIPE,71 )72 # cp = run(["python3", "getCandidateEdges.py", "-i", "best_by_pagerank_nodes.csv", "-d", distance, "-o", "candidate_edges.csv"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)73 # cp = run(["python3", "getCandidateEdges.py", "-i", "worst_by_pagerank_nodes.csv", "-d", distance, "-o", "candidate_edges.csv"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)74 log_file.write(f"getCandidateEdges.py{time() - temp_start}\n")75 log_file.write(f"{cp.stdout}\n")76 log_file.write(f"{cp.stderr}\n")77 log_file.write(f"{cp.returncode}\n")78 temp_start = time()79 # 3. Get positive edge sample.80 print("Get positive edge sample")81 run(["cp", path + "PythonFiles/getPositiveEdgeSample.py", "."])82 cp = run(83 [84 "python3",85 "getPositiveEdgeSample.py",86 "-g",87 "out_graph.txt",88 "-p",89 "100",90 "-o",91 "positive_edge_sample.csv",92 ],93 universal_newlines=True,94 stdout=subprocess.PIPE,95 stderr=subprocess.PIPE,96 )97 log_file.write(f"getPositiveEdgeSample.py{time() - temp_start}\n")98 log_file.write(f"{cp.stdout}\n")99 log_file.write(f"{cp.stderr}\n")100 log_file.write(f"{cp.returncode}\n")101 temp_start = time()102 # 4. Get negative edges sample.103 print("Get negative edge sample")104 run(["cp", path + "PythonFiles/getNegativeEdgeSample.py", "."])105 cp = run(106 [107 "python3",108 "getNegativeEdgeSample.py",109 "-g",110 "out_graph.txt",111 "-p",112 "100",113 "-o",114 "negative_edge_sample.csv",115 ],116 universal_newlines=True,117 stdout=subprocess.PIPE,118 stderr=subprocess.PIPE,119 )120 log_file.write(f"getNegativeEdgeSample.py{time() - temp_start}\n")121 log_file.write(f"{cp.stdout}\n")122 log_file.write(f"{cp.stderr}\n")123 log_file.write(f"{cp.returncode}\n")124 temp_start = time()125 # 5. Get node2vec node embeddings.126 print("Get node2vec node embeddings")127 run(["cp", path + "PythonFiles/getNodeEmbeddings.py", "."])128 cp = run(129 [130 "python3",131 "getNodeEmbeddings.py",132 "-g",133 "out_graph.txt",134 "-p",135 "node2vec",136 "--l",137 distance,138 "--pp",139 "1.0",140 "--qp",141 "1.0",142 "-o",143 "node2vec_node_embeddings.csv",144 ],145 universal_newlines=True,146 stdout=subprocess.PIPE,147 stderr=subprocess.PIPE,148 )149 log_file.write(f"getNodeEmbeddings.py{time() - temp_start}\n")150 log_file.write(f"{cp.stdout}\n")151 log_file.write(f"{cp.stderr}\n")152 log_file.write(f"{cp.returncode}\n")153 temp_start = time()154 # 6. Get fairwalk node embeddings.155 print("Get fairwalk node embeddings")156 cp = run(157 [158 "python3",159 "getNodeEmbeddings.py",160 "-g",161 "out_graph.txt",162 "-p",163 "fairwalk",164 "--l",165 distance,166 "--pp",167 "1.0",168 "--qp",169 "1.0",170 "-o",171 "fairwalk_node_embeddings.csv",172 ],173 universal_newlines=True,174 stdout=subprocess.PIPE,175 stderr=subprocess.PIPE,176 )177 log_file.write(f"getNodeEmbeddings.py{time() - temp_start}\n")178 log_file.write(f"{cp.stdout}\n")179 log_file.write(f"{cp.stderr}\n")180 log_file.write(f"{cp.returncode}\n")181 temp_start = time()182 # 7. Get candidates edges node2vec embeddings.183 print("Get node2vec edge embeddings")184 run(["cp", path + "PythonFiles/getEdgeEmbeddings.py", "."])185 cp = run(186 [187 "python3",188 "getEdgeEmbeddings.py",189 "-i",190 "node2vec_node_embeddings.csv",191 "-e",192 "candidate_edges.csv",193 "-p",194 "hadamart",195 "-o",196 "candidate_edges_node2vec_embeddings.csv",197 ],198 universal_newlines=True,199 stdout=subprocess.PIPE,200 stderr=subprocess.PIPE,201 )202 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")203 log_file.write(f"{cp.stdout}\n")204 log_file.write(f"{cp.stderr}\n")205 log_file.write(f"{cp.returncode}\n")206 temp_start = time()207 # 8. Get candidate edges fairwalk embeddings.208 print("Get fairwalk edge embeddings")209 cp = run(210 [211 "python3",212 "getEdgeEmbeddings.py",213 "-i",214 "fairwalk_node_embeddings.csv",215 "-e",216 "candidate_edges.csv",217 "-p",218 "hadamart",219 "-o",220 "candidate_edges_fairwalk_embeddings.csv",221 ],222 universal_newlines=True,223 stdout=subprocess.PIPE,224 stderr=subprocess.PIPE,225 )226 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")227 log_file.write(f"{cp.stdout}\n")228 log_file.write(f"{cp.stderr}\n")229 log_file.write(f"{cp.returncode}\n")230 temp_start = time()231 # 9. Get positive edge sample node2vec embeddings.232 print("Get positive edge sample node2vec embeddings")233 cp = run(234 [235 "python3",236 "getEdgeEmbeddings.py",237 "-i",238 "node2vec_node_embeddings.csv",239 "-e",240 "positive_edge_sample.csv",241 "-p",242 "hadamart",243 "-o",244 "positive_sample_edges_node2vec_embeddings.csv",245 ],246 universal_newlines=True,247 stdout=subprocess.PIPE,248 stderr=subprocess.PIPE,249 )250 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")251 log_file.write(f"{cp.stdout}\n")252 log_file.write(f"{cp.stderr}\n")253 log_file.write(f"{cp.returncode}\n")254 temp_start = time()255 # 10. Get positive edge sample fairwalk embeddings.256 print("Get positive edge sample fairwalk embeddings")257 cp = run(258 [259 "python3",260 "getEdgeEmbeddings.py",261 "-i",262 "fairwalk_node_embeddings.csv",263 "-e",264 "positive_edge_sample.csv",265 "-p",266 "hadamart",267 "-o",268 "positive_sample_edges_fairwalk_embeddings.csv",269 ],270 universal_newlines=True,271 stdout=subprocess.PIPE,272 stderr=subprocess.PIPE,273 )274 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")275 log_file.write(f"{cp.stdout}\n")276 log_file.write(f"{cp.stderr}\n")277 log_file.write(f"{cp.returncode}\n")278 temp_start = time()279 # 10. Get negative edge sample node2vec embeddings.280 print("Get negative edge sample node2vec embeddings")281 cp = run(282 [283 "python3",284 "getEdgeEmbeddings.py",285 "-i",286 "node2vec_node_embeddings.csv",287 "-e",288 "negative_edge_sample.csv",289 "-p",290 "hadamart",291 "-o",292 "negative_sample_edges_node2vec_embeddings.csv",293 ],294 universal_newlines=True,295 stdout=subprocess.PIPE,296 stderr=subprocess.PIPE,297 )298 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")299 log_file.write(f"{cp.stdout}\n")300 log_file.write(f"{cp.stderr}\n")301 log_file.write(f"{cp.returncode}\n")302 temp_start = time()303 # 11. Get negative edge sample fairwalk embeddings.304 print("Get negative edge sample fairwalk embeddings")305 cp = run(306 [307 "python3",308 "getEdgeEmbeddings.py",309 "-i",310 "fairwalk_node_embeddings.csv",311 "-e",312 "negative_edge_sample.csv",313 "-p",314 "hadamart",315 "-o",316 "negative_sample_edges_fairwalk_embeddings.csv",317 ],318 universal_newlines=True,319 stdout=subprocess.PIPE,320 stderr=subprocess.PIPE,321 )322 log_file.write(f"getEdgeEmbeddings.py{time() - temp_start}\n")323 log_file.write(f"{cp.stdout}\n")324 log_file.write(f"{cp.stderr}\n")325 log_file.write(f"{cp.returncode}\n")326 temp_start = time()327 # 12. Get node2vec classifier.328 print("Get node2vec classifier")329 run(["cp", path + "PythonFiles/getClassifier.py", "."])330 cp = run(331 [332 "python3",333 "getClassifier.py",334 "-p",335 "positive_sample_edges_node2vec_embeddings.csv",336 "-n",337 "negative_sample_edges_node2vec_embeddings.csv",338 "-o",339 "node2vec_recommender.sav",340 ],341 universal_newlines=True,342 stdout=subprocess.PIPE,343 stderr=subprocess.PIPE,344 )345 log_file.write(f"getClassifier.py{time() - temp_start}\n")346 log_file.write(f"{cp.stdout}\n")347 log_file.write(f"{cp.stderr}\n")348 log_file.write(f"{cp.returncode}\n")349 temp_start = time()350 # 13. Get fairwalk classifier.351 print("Get fairwalk classifier")352 cp = run(353 [354 "python3",355 "getClassifier.py",356 "-p",357 "positive_sample_edges_fairwalk_embeddings.csv",358 "-n",359 "negative_sample_edges_fairwalk_embeddings.csv",360 "-o",361 "fairwalk_recommender.sav",362 ],363 universal_newlines=True,364 stdout=subprocess.PIPE,365 stderr=subprocess.PIPE,366 )367 log_file.write(f"getClassifier.py{time() - temp_start}\n")368 log_file.write(f"{cp.stdout}\n")369 log_file.write(f"{cp.stderr}\n")370 log_file.write(f"{cp.returncode}\n")371 temp_start = time()372 # 14. Get red absorbing probabilities.373 print("Get red absorbing probabilites.")374 run(["cp", path + "PythonFiles/getRedAbsorbingProbs.py", "."])375 cp = run(376 [377 "python3",378 "getRedAbsorbingProbs.py",379 "-g",380 "out_graph.txt",381 "-o",382 "red_absorbing_probabilities.csv",383 ],384 universal_newlines=True,385 stdout=subprocess.PIPE,386 stderr=subprocess.PIPE,387 )388 log_file.write(f"getRedAbsorbingProbs.py{time() - temp_start}\n")389 log_file.write(f"{cp.stdout}\n")390 log_file.write(f"{cp.stderr}\n")391 log_file.write(f"{cp.returncode}\n")392 temp_start = time()393 # 15. Get Adamic Adar scores.394 print("Get adamic adar scores")395 run(["cp", path + "PythonFiles/getRecommendationScores.py", "."])396 cp = run(397 [398 "python3",399 "getRecommendationScores.py",400 "-i",401 "candidate_edges.csv",402 "-p",403 "adamic-adar",404 "-o",405 "adamic_adar_scores.csv",406 ],407 universal_newlines=True,408 stdout=subprocess.PIPE,409 stderr=subprocess.PIPE,410 )411 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")412 log_file.write(f"{cp.stdout}\n")413 log_file.write(f"{cp.stderr}\n")414 log_file.write(f"{cp.returncode}\n")415 temp_start = time()416 # 16. Get Jaccard coefficient scores.417 print("Get jaccard coefficient scores")418 cp = run(419 [420 "python3",421 "getRecommendationScores.py",422 "-i",423 "candidate_edges.csv",424 "-p",425 "jaccard-coefficient",426 "-o",427 "jaccard_coefficient_scores.csv",428 ],429 universal_newlines=True,430 stdout=subprocess.PIPE,431 stderr=subprocess.PIPE,432 )433 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")434 log_file.write(f"{cp.stdout}\n")435 log_file.write(f"{cp.stderr}\n")436 log_file.write(f"{cp.returncode}\n")437 temp_start = time()438 # 17. Get resource allocation scores.439 print("Get resource allocation scores")440 cp = run(441 [442 "python3",443 "getRecommendationScores.py",444 "-i",445 "candidate_edges.csv",446 "-p",447 "resource-allocation",448 "-o",449 "resource_allocation_scores.csv",450 ],451 universal_newlines=True,452 stdout=subprocess.PIPE,453 stderr=subprocess.PIPE,454 )455 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")456 log_file.write(f"{cp.stdout}\n")457 log_file.write(f"{cp.stderr}\n")458 log_file.write(f"{cp.returncode}\n")459 temp_start = time()460 # 18. Get preferencial attachment scores.461 print("Get prerferential attachment scores.")462 cp = run(463 [464 "python3",465 "getRecommendationScores.py",466 "-i",467 "candidate_edges.csv",468 "-p",469 "preferential-attachment",470 "-o",471 "preferential_attachment_scores.csv",472 ],473 universal_newlines=True,474 stdout=subprocess.PIPE,475 stderr=subprocess.PIPE,476 )477 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")478 log_file.write(f"{cp.stdout}\n")479 log_file.write(f"{cp.stderr}\n")480 log_file.write(f"{cp.returncode}\n")481 temp_start = time()482 # 19. Get node2vec scores.483 print("Get node2vec scores.")484 cp = run(485 [486 "python3",487 "getRecommendationScores.py",488 "-i",489 "candidate_edges_node2vec_embeddings.csv",490 "-p",491 "from-classifier",492 "-c",493 "node2vec_recommender.sav",494 "-o",495 "node2vec_scores.csv",496 ],497 universal_newlines=True,498 stdout=subprocess.PIPE,499 stderr=subprocess.PIPE,500 )501 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")502 log_file.write(f"{cp.stdout}\n")503 log_file.write(f"{cp.stderr}\n")504 log_file.write(f"{cp.returncode}\n")505 temp_start = time()506 # 20. Get fairwalk scores.507 print("Get fairwalk scores.")508 cp = run(509 [510 "python3",511 "getRecommendationScores.py",512 "-i",513 "candidate_edges_fairwalk_embeddings.csv",514 "-p",515 "from-classifier",516 "-c",517 "fairwalk_recommender.sav",518 "-o",519 "fairwalk_scores.csv",520 ],521 universal_newlines=True,522 stdout=subprocess.PIPE,523 stderr=subprocess.PIPE,524 )525 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")526 log_file.write(f"{cp.stdout}\n")527 log_file.write(f"{cp.stderr}\n")528 log_file.write(f"{cp.returncode}\n")529 temp_start = time()530 # 21. Get fair scores.531 print("Get fair scores.")532 cp = run(533 [534 "python3",535 "getRecommendationScores.py",536 "-i",537 "candidate_edges.csv",538 "-p",539 "fair",540 "-o",541 "fair_scores.csv",542 ],543 universal_newlines=True,544 stdout=subprocess.PIPE,545 stderr=subprocess.PIPE,546 )547 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")548 log_file.write(f"{cp.stdout}\n")549 log_file.write(f"{cp.stderr}\n")550 log_file.write(f"{cp.returncode}\n")551 temp_start = time()552 # 21.b Get sum fair scores.553 print("Get sum fair scores.")554 cp = run(555 [556 "python3",557 "getRecommendationScores.py",558 "-i",559 "candidate_edges.csv",560 "-p",561 "sum-fair",562 "-o",563 "sum_fair_scores.csv",564 ],565 universal_newlines=True,566 stdout=subprocess.PIPE,567 stderr=subprocess.PIPE,568 )569 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")570 log_file.write(f"{cp.stdout}\n")571 log_file.write(f"{cp.stderr}\n")572 log_file.write(f"{cp.returncode}\n")573 temp_start = time()574 # 21.c Get dyadic fair scores.575 print("Get dyadic fair scores.")576 cp = run(577 [578 "python3",579 "getRecommendationScores.py",580 "-i",581 "candidate_edges.csv",582 "-p",583 "dyadic-fair",584 "-o",585 "dyadic_fair_scores.csv",586 ],587 universal_newlines=True,588 stdout=subprocess.PIPE,589 stderr=subprocess.PIPE,590 )591 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")592 log_file.write(f"{cp.stdout}\n")593 log_file.write(f"{cp.stderr}\n")594 log_file.write(f"{cp.returncode}\n")595 temp_start = time()596 # 22. Get multiplicative hybrid node2vec scores.597 print("Get hybrid scores.")598 cp = run(599 [600 "python3",601 "getRecommendationScores.py",602 "-i",603 "candidate_edges.csv",604 "-p",605 "multiplicative-hybrid",606 "-f",607 "fair_scores.csv",608 "-c",609 "node2vec_scores.csv",610 "-o",611 "hybrid_node2vec_scores.csv",612 ],613 universal_newlines=True,614 stdout=subprocess.PIPE,615 stderr=subprocess.PIPE,616 )617 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")618 log_file.write(f"{cp.stdout}\n")619 log_file.write(f"{cp.stderr}\n")620 log_file.write(f"{cp.returncode}\n")621 temp_start = time()622 log_file.write(f"Total: {time() - start}\n")623 # 22.b Get multiplicative hybrid node2vec scores.624 print("Get balanced hybrid scores.")625 cp = run(626 [627 "python3",628 "getRecommendationScores.py",629 "-i",630 "candidate_edges.csv",631 "-p",632 "multiplicative-hybrid",633 "-f",634 "dyadic_fair_scores.csv",635 "-c",636 "node2vec_scores.csv",637 "-o",638 "hybrid_balanced_node2vec_scores.csv",639 ],640 universal_newlines=True,641 stdout=subprocess.PIPE,642 stderr=subprocess.PIPE,643 )644 log_file.write(f"getRecommendationScores.py{time() - temp_start}\n")645 log_file.write(f"{cp.stdout}\n")646 log_file.write(f"{cp.stderr}\n")647 log_file.write(f"{cp.returncode}\n")648 temp_start = time()649 log_file.write(f"Total: {time() - start}\n")650 # 23. Run Experiment Score and Acceptance probabilities651 run(["cp", path + "ExperimentScripts/experiment_one_fairness.py", "."])652 run(["cp", path + "ExperimentScripts/experiment_two_acceptance.py", "."])653 run(["cp", path + "ExperimentScripts/experiment_three_personalized.py", "."])654 scores_files = [655 "fair",656 "adamic_adar",657 "jaccard_coefficient",658 "resource_allocation",659 "preferential_attachment",660 "node2vec",661 "fairwalk",662 "hybrid_node2vec",663 "dyadic_fair",664 "hybrid_balanced_node2vec",665 ] # , "sum_fair"666 print("Run experiment one.")667 for sf in scores_files:668 cp = run(669 [670 "python3",671 "experiment_one_fairness.py",672 "-r",673 rounds,674 "-s",675 f"{sf}_scores.csv",676 "-o",677 f"sc_{sf}.csv",678 ],679 universal_newlines=True,680 stdout=subprocess.PIPE,681 stderr=subprocess.PIPE,682 )683 log_file.write(f"experiment_one_fairness -> policy: {sf}\n")684 log_file.write(f"{cp.stdout}\n")685 log_file.write(f"{cp.stderr}\n")686 log_file.write(f"{cp.returncode}\n")687 print("Run experiment two.")688 for sf in scores_files:689 cp = run(690 [691 "python3",692 "experiment_two_acceptance.py",693 "-r",694 rounds,695 "-s",696 f"{sf}_scores.csv",697 "-n",698 "node2vec_scores.csv",699 "-o",700 f"accept_prob_{sf}.csv",701 ],702 universal_newlines=True,703 stdout=subprocess.PIPE,704 stderr=subprocess.PIPE,705 )706 log_file.write(f"experiment_two_fairness -> policy: {sf}\n")707 log_file.write(f"{cp.stdout}\n")708 log_file.write(f"{cp.stderr}\n")709 log_file.write(f"{cp.returncode}\n")710 print("Run experiment three.")711 for sf in scores_files:712 cp = run(713 [714 "python3",715 "experiment_three_personalized.py",716 "-r",717 rounds,718 "-s",719 f"{sf}_scores.csv",720 "-o",721 f"sc_personalized_{sf}.csv",722 ],723 universal_newlines=True,724 stdout=subprocess.PIPE,725 stderr=subprocess.PIPE,726 )727 log_file.write(f"experiment_two_fairness -> policy: {sf}\n")728 log_file.write(f"{cp.stdout}\n")729 log_file.write(f"{cp.stderr}\n")730 log_file.write(f"{cp.returncode}\n")731 # 24 deletion edges.732 print("Get deletion scores.")733 run(["cp", path + "/CppFiles/get_deletion_scores.out", "."])...

upload_log_rsi_copy.py

Source:upload_log_rsi_copy.py

1#!/usr/bin/env python32import pexpect3import getpass4import os5import time6import d_list7import sys8import re9import subprocess10from slacker import Slacker11from netrc import netrc12####################################### get_sysargv() #################################13def get_sysargv():14 log = []15 tmp_30 = '#'*3016 UN = 'cne'17 #PW = getpass.getpass("Password for %s: "% UN)18 with open('%s/.cne' % os.path.expanduser('~'), 'rU') as fd: PW = fd.read()19 if len(sys.argv)<2:20 tmp_str = 'Please, specify at least one FWs hostname as an argument. Example: ./get_data.py LB3-CFW'21 print(tmp_str)22 sys.exit(1)23 elif len(sys.argv) == 2 and sys.argv[1] in d_list.devices:24 hn = sys.argv[1]25 device_cred = {'device_hn': hn, 'device_ip': d_list.devices[hn], 'device_un': UN, 'device_pw': PW, 'case_num': None, 'slack_ch': None}26 sr_num = '--ABSENT--'27 28 log.append(tmp_30)29 tmp_str_1 = 'Case number was not provided, files will be dumped locally'30 log.append(tmp_str_1)31 tmp_str_2 = 'Working with %s, %s' % (sys.argv[1], d_list.devices[sys.argv[1]] )32 log.append(tmp_str_2)33 return device_cred, log34 elif len(sys.argv) == 3:35 for i in sys.argv[1:]:36 if i in d_list.devices:37 hn = i38 elif re.search(r'[0-9]+-[0-9]+-[0-9]+', i):39 sr_num = i40 else:41 tmp_str = 'Cannot recognize args. Example: ./get_data.py LB3-CFW 2017-1117-25252'42 print(tmp_str)43 sys.exit(1)44 device_cred = {'device_hn': hn, 'device_ip': d_list.devices[hn], 'device_un': UN, 'device_pw': PW, 'case_num': sr_num, 'slack_ch': None}45 elif len(sys.argv) == 4:46 for i in sys.argv[1:]:47 if i in d_list.devices:48 hn = i49 elif re.search(r'[0-9]+-[0-9]+-[0-9]+', i):50 sr_num = i51 elif re.search(r'--', i):52 sl_str = re.search(r'--(.*)', i).group(1)53 sl_ch = sl_str54 else:55 tmp_str = 'Cannot recognize args. Example: ./get_data.py LB3-CFW 2017-1117-25252 --#network-active'56 print(tmp_str)57 sys.exit(1)58 device_cred = {'device_hn': hn, 'device_ip': d_list.devices[hn], 'device_un': UN, 'device_pw': PW, 'case_num': sr_num, 'slack_ch': sl_ch}59 else:60 tmp_str = 'No match, specify at least a correct FWs hostname as an argument. Example: ./get_data.py LB3-CFW'61 print(tmp_str)62 sys.exit(1)63 64 tmp_str_4 = 'Case number is {}, files will be uploaded automatically'.format(sr_num)65 tmp_str_5 = 'Working with %s, %s' % (hn, d_list.devices[hn])66 log.append(tmp_30)67 log.append(tmp_str_4)68 log.append(tmp_str_5)69 return device_cred, log70##################################################################################################71################################### which one is Primary??? #####################################72def determine_prim(prompt, log_file):73 if re.search(r'{primary:node([0-1])}', str(prompt)):74 node_num = re.search(r'{primary:node([0-1])}', str(prompt)).group(1)75 else:76 with open(log_file, 'a') as fd: fd.write("\ndon't see the prompts, exiting")77 sys.exit(1)78 if node_num == '0':79 prim_node = 'node0'80 sec_node = 'node1'81 elif node_num == '1':82 prim_node = 'node1'83 sec_node = 'node0'84 else:85 with open(log_file, 'a') as fd: fd.write("\ndon't recoznize the node num, exiting")86 sys.exit(1)87 with open(log_file, 'a') as fd: fd.write("\nPrimary node is {}".format(prim_node))88 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)89 with open(log_file, 'rU') as fd: print(fd.read())90 return prim_node, sec_node91#################################################################################################92############## clean unicode string from /r/n/r and crap like that ######################93def clean_unicode(uni_str):94 str_tmp = re.sub(r'\\r\\n\\r', '\n', uni_str)95 str_pretty = re.sub(r'\\r', '\n', str_tmp)96 return str_pretty97#################################################################################################98######################################### tgz_logs ################################99###### leaves 2 archived files on the box and returns their names for further scp-ing ###########100def tgz_logs(device_cred, time_str, log_file):101 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)102 with open(log_file, 'a') as fd: fd.write('\n' + 'Working on getting logs\n')103 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)104 child = pexpect.spawn('ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ' + device_cred['device_un'] + '@' + device_cred['device_ip'])105 try: 106 child.expect('Password:')107 child.sendline(device_cred['device_pw'])108 user_at_host_string = device_cred['device_un'] + '@' + device_cred['device_hn']109 child.expect(user_at_host_string)110 prim_node, sec_node = determine_prim(child.before, log_file)111 112 fn_sec = device_cred['device_hn'] + '_var_log_' + sec_node + '_' + time_str + '.tgz'113 fn_prim = device_cred['device_hn'] + '_var_log_' + prim_node + '_' + time_str + '.tgz'114 any_node = 'node*'115 fn_scp = device_cred['device_hn'] + '_var_log_' + any_node + '_' + time_str + '.tgz'116 child.sendline('file archive compress source /var/log/ destination /var/tmp/' + fn_prim)117 child.expect(user_at_host_string)118 child.sendline('start shell')119 child.expect('%')120######### in case if Secondary is dead #########121 try: 122 child.sendline('rlogin -T ' + sec_node)123 child.expect('secondary', timeout = 5)124 child.sendline('file archive compress source /var/log/ destination /var/tmp/' + fn_sec)125 child.expect('secondary')126 child.sendline('file copy /var/tmp/' + fn_sec + ' ' + prim_node + ':/var/tmp/')127 child.expect('secondary')128 child.close()129 return fn_prim, fn_sec, fn_scp130 except:131 with open(log_file, 'a') as fd: fd.write('\n########\nException was raised\nSecondary is unavailable!\n###########')132 child.close()133 with open(log_file, 'rU') as fd: print(fd.read())134 return fn_prim, fn_prim, fn_scp135###############################################136 except Exception as var:137 with open(log_file, 'a') as fd: fd.write('\n' + str(var))138 with open(log_file, 'a') as fd: fd.write('\nException was raised!')139 sys.exit(1)140################################################################################################141######################## get_logs! ########################################################142####### scp-s files left by previous function to the box ####################################143def get_logs(fn_prim, fn_sec, fn_scp, device_cred, log_file):144 dird = '/root/data_to_upload/' + device_cred['device_hn']145#### creating directory #####146 if not os.path.exists(dird):147 os.mkdir(dird)148#############################149 dest_path = device_cred['device_un'] + '@' + device_cred['device_ip'] + r':/var/tmp/'150 scp_str = 'scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ' + dest_path + fn_scp + r' ' + dird151 with open(log_file, 'a') as fd: fd.write('\n' + scp_str)152 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)153 child = pexpect.spawn(scp_str)154 try:155 child.expect('Password:')156 child.sendline(device_cred['device_pw'])157 child.expect([pexpect.TIMEOUT, pexpect.EOF], timeout = 1200)158 download_str =clean_unicode(str(child.before)[2:-1])159 child.close()160 with open(log_file, 'a') as fd: fd.write('\n' + download_str)161 if fn_prim != fn_sec: ls_str = 'ls -l ' + dird + ' | grep "' + fn_prim + r'\|' + fn_sec + '"'162 else: ls_str = 'ls -l ' + dird + ' | grep ' + fn_prim163 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)164 with open(log_file, 'a') as fd: fd.write('\n' + ls_str)165 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)166 #os.system(ls_str)167 ls = subprocess.Popen(ls_str, stdout=subprocess.PIPE, shell=True)168 byte_output = ls.communicate()[0]169 str_output = byte_output.decode('ASCII')170 with open(log_file, 'a') as fd: fd.write('\n' + str_output)171 with open(log_file, 'rU') as fd: print(fd.read())172 except Exception as var:173 with open(log_file, 'a') as fd: fd.write('\n' + str(var))174 with open(log_file, 'a') as fd: fd.write('\n' + 'Exception was raised!')175 sys.exit(1)176 177##################################################################################################178############################### upload to the casemanager ###################################179def upload(dev_cred, fn_prim, fn_sec, log_file):180 dird = '/root/data_to_upload/' + dev_cred['device_hn']181 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)182 with open(log_file, 'a') as fd: fd.write('\n' + 'Connecting to JTAC server')183 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)184 child = pexpect.spawn('sftp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no anonymous@sftp.juniper.net')185 try:186 child.expect('assword:')187 child.sendline('anonymous')188 child.expect('sftp>')189 with open(log_file, 'a') as fd: fd.write('\n' + 'Got in!!')190 child.sendline('mkdir pub/incoming/' + dev_cred['case_num'])191 child.expect('sftp>', timeout = 900)192 child.sendline('cd pub/incoming/' + dev_cred['case_num'])193 child.expect('sftp>', timeout = 900)194 child.sendline('put ' + dird + '/' + fn_prim)195 child.expect('sftp>', timeout = 1200)196 download_str_1 = clean_unicode(str(child.before)[2:-1])197 if fn_prim != fn_sec:198 child.sendline('put ' + dird + '/' + fn_sec)199 child.expect('sftp>', timeout = 1200)200 download_str_2 = clean_unicode(str(child.before)[2:-1])201 child.close()202 else: 203 download_str_2 = ''204 child.close()205 if 'Permission denied' in download_str_1 or 'Permission denied' in download_str_2:206 with open(log_file, 'a') as fd: fd.write('\n' + '###########\n\n\n\n\nSomething went wrong! Files were not copied \n\n\n########')207 sys.exit(1)208 else: 209 with open(log_file, 'a') as fd: fd.write('\n' + download_str_1)210 with open(log_file, 'a') as fd: fd.write('\n' + download_str_2)211 with open(log_file, 'rU') as fd: print(fd.read())212 except Exception as var:213 with open(log_file, 'a') as fd: fd.write('\n' + '###########\n\n\n\n\nSomething went wrong! Files were not copied \n\n\n########')214 with open(log_file, 'a') as fd: fd.write('\n' + str(var))215################################################################################################216###################################### get RSI #########################################217def get_rsi(device_cred, time_str, log_file):218 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)219 with open(log_file, 'a') as fd: fd.write('\n' + 'Working on getting RSI, be patient :-)\n')220 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)221 child = pexpect.spawn('ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ' + device_cred['device_un'] + '@' + device_cred['device_ip'])222 try:223 child.expect('Password:')224 child.sendline(device_cred['device_pw'])225 user_at_host_string = device_cred['device_un'] + '@' + device_cred['device_hn']226 child.expect(user_at_host_string)227 prim_node, sec_node = determine_prim(child.before, log_file)228 rsi_name = device_cred['device_hn'] + '_RSI_' + prim_node + '_' + time_str + '.txt'229 rsi_name_arch = device_cred['device_hn'] + '_RSI_' + prim_node + '_' + time_str + '.tgz'230 with open(log_file, 'a') as fd: fd.write('\n' + "Now waiting for RSI to complete... Up to 90 min")231 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)232 child.sendline('request support information | save /var/tmp/' + rsi_name)233 try:234 child.expect('lines of output to', timeout = 5400)235 with open(log_file, 'a') as fd: fd.write('\n' + 'RSI completed: /var/tmp/%s' % rsi_name)236 with open(log_file, 'a') as fd: fd.write('\n' + '#'*30)237 child.close()238 with open(log_file, 'rU') as fd: print(fd.read())239 return rsi_name, rsi_name, rsi_name240 except pexpect.TIMEOUT:241 with open(log_file, 'a') as fd: fd.write('\n' + '###########\n\n\n\n\nIt takes too long! Timeout received. Will be uploading what I have... \n\n\n########')242 return rsi_name, rsi_name, rsi_name243 except Exception as var:244 with open(log_file, 'a') as fd: fd.write('\n' + '###########\n\n\n\n\nSomething went wrong! Files were not copied \n\n\n########')245 with open(log_file, 'a') as fd: fd.write('\n' + str(var))246 sys.exit(1)247 except Exception as var:248 with open(log_file, 'a') as fd: fd.write('\n' + '###########\n\n\n\n\nSomething went wrong! Files were not copied \n\n\n########')249 with open(log_file, 'a') as fd: fd.write('\n' + str(var))250 sys.exit(1)251############################### main ######################################################252def fetch_logs(device_cred, time_str, log_file):253 fn_prim, fn_sec, fn_scp = tgz_logs(device_cred, time_str, log_file)254 get_logs(fn_prim, fn_sec, fn_scp, device_cred, log_file)255 if device_cred['case_num'] != None:256 upload(device_cred, fn_prim, fn_sec, log_file)257#######################258def fetch_rsi(device_cred, time_str, log_file):259 fn_prim, fn_sec, fn_scp = get_rsi(device_cred, time_str, log_file) 260 get_logs(fn_prim, fn_sec, fn_scp, device_cred, log_file)261 262 if device_cred['case_num'] != None:263 upload(device_cred, fn_prim, fn_sec, log_file)264#################################################################################################265def get_creds():266 creds = netrc('%s/.netrc' % os.path.expanduser('~'))267 return creds268def send_slack(message, channel_id, log_file):269 creds = get_creds()270 username, account, password = creds.authenticators('slackbot')271 #slack = Slacker(password)272 #response = slack.chat.post_message(as_user = True, text=message, channel = channel_id)273 curl_str = 'curl -F file=@%s -F token=%s -F channels=%s https://slack.com/api/files.upload' % (log_file, password, channel_id)274 print(curl_str)275 os.system(curl_str)276################## main #####################################################################277def main():278 time_str = time.strftime('%Y-%m-%d_%H-%M', time.localtime())279 280 device_cred, log = get_sysargv()281 282 dird = '/root/data_to_upload/' + device_cred['device_hn']283#### creating directory #####284 if not os.path.exists(dird):285 os.mkdir(dird)286#############################287 log_file = dird + '/' + device_cred['device_hn']+ '_execution-log_' + time_str + '.log'288 print('Execution log: %s' % log_file)289 try: os.remove(log_file)290 except: pass291 log_string = '\n'.join(log)292 with open(log_file, 'a') as fd: fd.write(log_string)293 print(log_string)294 fetch_logs(device_cred, time_str, log_file)295 fetch_rsi(device_cred, time_str, log_file)296 297 with open(log_file, 'rU') as fd: log_message = fd.read()298 299 print(log_message)300 301 if device_cred['slack_ch'] != None:302 channel_id = device_cred['slack_ch']303 send_slack(log_message, channel_id, log_file)304#################################################################################################305if __name__ == '__main__':...

server.py

Source:server.py

1import socket2import os3import threading,sys4sock = socket.socket()5file=open('Users.txt',"a")6file.close()7file=open('log.txt',"w")8file.close()9def path_(sentence):10 if os.name == 'posix':11 if sentence.find('/') == -1:12 return True13 else:14 return False15 elif os.name == 'nt':16 if sentence.find('\\') == -1:17 return True18 else:19 return False20def cat_(name):21 try:22 file = open(name)23 content = file.read()24 file.close()25 return content26 except FileNotFoundError:27 log_file = open('log.txt', 'a')28 log_file.write('File doesnt exist' + "\n")29 log_file.close()30 return "File doesnt exist"31def echo_(sentence, directory):32 try:33 if len(sentence.split(' >> ')) == 2:34 sentence2= sentence.split(' >> ')35 sentence2[1] = os.path.join(directory, sentence2[1])36 file = open(sentence2[1], 'a')37 file.write('\n')38 file.write(sentence2[0])39 file.close()40 log_file = open('log.txt', 'a')41 log_file.write('Text is written in' + str(sentence2[1]) + "\n")42 log_file.close()43 return f'Text is written in {sentence2[1]}'44 elif len(sentence.split(' > ')) == 2:45 sentence2 = sentence.split(' > ')46 sentence2[1] = os.path.join(directory, sentence2[1])47 file = open(sentence2[1], 'w')48 file.write('\n')49 file.write(sentence2[0])50 file.close()51 log_file = open('log.txt', 'a')52 log_file.write('Text is written in' + str(sentence2) + "\n")53 log_file.close()54 return f'Text is written in {sentence2[1]}'55 else:56 log_file = open('log.txt', 'a')57 log_file.write('Wrong format'+ "\n")58 log_file.close()59 return 'Wrong format'60 except FileNotFoundError:61 log_file = open('log.txt', 'a')62 log_file.write('Wrong format' + "\n")63 log_file.close()64 return 'Wrong format'65def mkdir_(sentence, directory):66 if sentence[0] == ' ':67 log_file = open('log.txt', 'a')68 log_file.write('Wrong format' + "\n")69 log_file.close()70 return ('Wrong format')71 else:72 try:73 os.makedirs(os.path.join(directory, sentence))74 return os.path.join(directory, sentence)75 except OSError:76 log_file = open('log.txt', 'a')77 log_file.write('Folder already exists' + "\n")78 log_file.close()79 return ('Folder already exists')80def rm_(sentence, directory):81 if sentence[0] == ' ':82 log_file = open('log.txt', 'a')83 log_file.write('Wrong format' + "\n")84 log_file.close()85 return ('Wrong format')86 else:87 try:88 os.remove(os.path.join(directory, sentence))89 return os.path.join(directory, sentence)90 except FileNotFoundError:91 log_file = open('log.txt', 'a')92 log_file.write('File already exists' + "\n")93 log_file.close()94 return ('File already exists')95 except PermissionError:96 log_file = open('log.txt', 'a')97 log_file.write('ERROR' + "\n")98 log_file.close()99 return ('ERROR')100def rmdir_(sentence, directory):101 if sentence[0] == ' ':102 log_file = open('log.txt', 'a')103 log_file.write('Wrong format' + "\n")104 log_file.close()105 return ('Wrong format')106 else:107 try:108 os.rmdir(os.path.join(directory, sentence))109 return os.path.join(directory,sentence)110 except FileNotFoundError:111 log_file = open('log.txt', 'a')112 log_file.write('Folder doesnt exist' + "\n")113 log_file.close()114 return ('Folder doesnt exist')115def cd_(sentence, directory):116 if sentence == '..':117 now_dir = os.path.split(directory)[0]118 return (now_dir)119 elif sentence[0] == ' ' or sentence[len(sentence) - 1] == ' ':120 log_file = open('log.txt', 'a')121 log_file.write('Wrong format' + "\n")122 log_file.close()123 return ('Wrong format')124 elif os.path.exists(os.path.join(directory, sentence)):125 if path_(sentence):126 return os.path.join(directory, sentence)127 else:128 log_file = open('log.txt', 'a')129 log_file.write('ERROR' + "\n")130 log_file.close()131 return 'ERROR'132 else:133 log_file = open('log.txt', 'a')134 log_file.write('Directory doesnt exist' + "\n")135 log_file.close()136 return 'Directory doesnt exist'137def send_(conn, msg):138 try:139 header = f'{len(msg):<4}'140 conn.send(f'{header}{msg}'.encode())141 except ConnectionAbortedError:142 pass143def recv_(conn):144 try:145 header = int(conn.recv(4).decode().strip())146 data = conn.recv(header * 2).decode()147 return data148 except (ValueError, ConnectionAbortedError):149 return150class Server(threading.Thread):151 def __init__(self, conn, addr):152 super().__init__()153 self.conn = conn154 self.addr = addr155 self.dir = ''156 self.during_dir = ''157 def run(self):158 pr=True159 avtorise = recv_(self.conn)160 name = recv_(self.conn)161 try:162 if avtorise==1:163 with open('Users.txt', 'r') as f:164 text = f.read()165 if name not in text:166 pr = True167 else:168 pr= False169 else:170 file = open('Users.txt', 'r')171 text = file.read()172 if name in text:173 pr = True174 except KeyError:175 pr = False176 send_(self.conn, str(pr))177 while not pr:178 name = recv_(self.conn)179 pr = True180 try:181 if avtorise == 1:182 with open('Users.txt', 'r') as f:183 text = f.read()184 if name not in text:185 pr = True186 else:187 pr = False188 else:189 file = open('Users.txt', 'r')190 text = file.read()191 if name in text:192 pr = True193 except KeyError:194 pr = False195 send_(self.conn, str(pr))196 # Проверка пароля197 pr = False198 pswd = recv_(self.conn)199 if avtorise == '1':200 with open('Users.txt', 'a') as f:201 f.write(name + ';')202 f.write(pswd + '\n')203 pr = True204 else:205 with open('Users.txt', 'r') as f:206 for line in f:207 symb = line.split(';')208 if symb[0] == name:209 if symb[1] == pswd + '\n':210 pr = True211 send_(self.conn, str(pr))212 while not pr:213 pswd = recv_(self.conn)214 with open('Users.txt', 'r') as f:215 for line in f:216 symb = line.split(';')217 if symb[0] == name:218 if symb[1] == pswd + '\n':219 pr = True220 send_(self.conn, str(pr))221 self.dir = os.path.join(os.getcwd(), name)222 self.during_dir = os.path.join(os.getcwd(), name)223 mkdir_(name, os.getcwd())224 while True:225 request = recv_(self.conn)226 log_file = open('log.txt', 'a')227 log_file.write(str(request) + "\n")228 log_file.close()229 print(request)230 if request:231 if request == 'pwd':232 send_(self.conn, self.during_dir)233 elif request == 'ls':234 send_(self.conn, '; '.join(os.listdir(self.during_dir)))235 elif request.split(' ')[0] == 'cat':236 try:237 send_(self.conn, cat_(os.path.join(self.during_dir, request.split(' ')[1])))238 except IndexError:239 send_(self.conn, "This file doesnt exit")240 elif request[:5] == 'echo ':241 send_(self.conn, echo_(request[5:], self.during_dir))242 elif request[:6] == 'mkdir ':243 if path_(request[6:]):244 self.during_dir = mkdir_(request[6:], self.during_dir)245 send_(self.conn, self.during_dir)246 else:247 send_(self.conn, 'You cant enter the way')248 elif request[:3] == 'rm ':249 if path_(request[3:]):250 send_(self.conn, rm_(request[3:], self.during_dir))251 else:252 send_(self.conn, 'You cant enter the way')253 elif request[:6] == 'rmdir ':254 if path_(request[:6]):255 send_(self.conn, rmdir_(request[6:], self.during_dir))256 else:257 send_(self.conn, 'You cant enter the way')258 elif request[:3] == 'cd ':259 if (self.during_dir == self.dir and request[3:] != '..') or (self.during_dir != self.dir):260 x = cd_(request[3:], self.during_dir)261 if x != 'Wrong format' or x != 'Directory doesnt exist' or x != 'You cant enter the way':262 self.during_dir = x263 send_(self.conn, self.during_dir)264 else:265 send_(self.conn, x)266 elif (self.during_dir == self.dir and request[3:] == '..'):267 send_(self.conn, 'You cant log out of your directory')268 else:269 send_(self.conn, 'Request ERROR')270 elif request is None:271 break272 else:273 send_(self.conn, 'Request ERROR')274 self.conn.close()275log_file = open('log.txt', 'a')276print('Start...')277log_file.write('Start...'+"\n")278port = 9090279print('Connection...')280log_file.write('Connection...'+"\n")281sock.bind(('', port))282print('Port> ',port)283log_file.write('Port> '+ str(port)+"\n")284print('Start listening...')285log_file.write('Start listening...'+"\n")286sock.listen()287log_file.close()288while True:289 conn, addr = sock.accept()290 print('Client>',addr[0])...

Show more

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

• JUnit Tutorial
• TestNG Tutorial
• Webdriver Tutorial
• WebDriverIO Tutorial
• Protractor Tutorial
• Selenium 4 Tutorial
• Jenkins Tutorial
• NUnit Tutorial
• Jest Tutorial
• Playwright Tutorial
• Cypress Tutorial
• PyTest Tutorial

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Robotframework automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

View sample repo View DocumentationSign up Free

›

_