How to use get_output_report method in prospector

Best Python code snippet using prospector_python

uap_statistic.py

Source:uap_statistic.py

...10import MailClient11from common_data import HADOOP_CLIENT,dag_list,dag_input_dict,dag_log_dict,dag_output_dict,input_size_dict,output_size_dict, yes_input_size_dict,yes_output_size_dict,delta_input_size_dict,delta_output_size_dict,dict_map_input_bytes,dict_slot_time,delta_slot_time,delta_map_input_bytes12import common_lib13from common_lib import get_slot_time,init_structure,print_dict,mark_data,data_to_str14def get_output_report(base_time):15    time_format = base_time.split('-')16    year        = time_format[0]17    month       = time_format[1]18    day         = time_format[2]19    20    mkdate = datetime.date(int(year),int(month), int(day))21    yesterday = str(mkdate - datetime.timedelta(days = 1))22    tomorrow  = str(mkdate + datetime.timedelta(days=1))23    yes_yes   = str(mkdate - datetime.timedelta(days = 2))24    yes       = yesterday.replace('-','')25    day_time  = base_time.replace('-','')26    yes_yes = yes_yes.replace('-','')27    print "yes_yes " + str(yes_yes)28    #print str(yes)+"--" + str(day_time)29    30#    print "start_time " + str(start_time)31#    print "end_time" + str(end_time)32    for dag in dag_list:33        output_size_dict[dag]       = 034        yes_output_size_dict[dag]   = 035        delta_output_size_dict[dag] = 036        37    38    for dag in dag_list:39        log_path       = dag_output_dict[dag]40        for log in log_path:41            ls_cmd     = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/event_day=" + day_time +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'42            yes_ls_cmd = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/event_day=" + yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'43            #handle holmes44            if dag == 'udwetl_holmes4uap':45                ls_cmd     = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/event_day=" + yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'46                yes_ls_cmd = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/event_day=" + yes_yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'47            48            print "current yes command is :" + str(yes_ls_cmd)49            print "current command is :" + str(ls_cmd)50            ls_cmd_re  = os.popen(ls_cmd).readlines()51            52            if ls_cmd_re:53                print "In "+dag+ " " + str(log)+" not exist "54                ls_cmd_re    = []55            else:56                print "In "+dag+ " " + str(log)+"  exist "57                ls_cmd_re    = []58                get_size_cmd = HADOOP_CLIENT + " fs -dus " + log.strip() + "/event_day=" + day_time + " | awk '{print $2}'"59                #handle holmes60                if dag == 'udwetl_holmes4uap':61                    get_size_cmd = HADOOP_CLIENT + " fs -dus " + log.strip() + "/event_day=" + yes + " | awk '{print $2}'"                 62                print get_size_cmd63                get_size_cmd_re          = os.popen(get_size_cmd).readlines()64                print type(get_size_cmd_re)65                log_size                 = str(get_size_cmd_re[0])66                print  type(log_size)67                log_size                 = str(log_size).replace('\n','')68                print log_size69                ToGB                     = float(log_size)/(1024*1024*1024)70                ToGB                     = round(ToGB,2)71                print "current :" + str(ToGB) + "GB"72                if output_size_dict.has_key(dag):73                    output_size_dict[dag] = output_size_dict[dag] + ToGB74                    print str(dag) + " Total size :" + str(output_size_dict[dag]) + "--second time"75                else:76                    output_size_dict[dag] = ToGB77                    print str(dag) + " Total size : " + str(output_size_dict[dag]) + "--first time"78            yes_ls_cmd_re     = os.popen(yes_ls_cmd).readlines()79            if yes_ls_cmd_re:80                print "In "+dag+ " " + str(log)+" not exist "81                yes_ls_cmd_re = []82            else:83                print "In "+dag+ " " + str(log)+"  exist "84                yes_ls_cmd_re = []85                get_size_cmd  = HADOOP_CLIENT + " fs -dus " + log.strip() + "/event_day=" + yes + " | awk '{print $2}'"86                #handle holmes87                if dag == 'udwetl_holmes4uap':88                    get_size_cmd = HADOOP_CLIENT + " fs -dus " + log.strip() + "/event_day=" + yes_yes + " | awk '{print $2}'"89                    90                print get_size_cmd91                get_size_cmd_re               = os.popen(get_size_cmd).readlines()92                print type(get_size_cmd_re)93                log_size                      = str(get_size_cmd_re[0])94                print  type(log_size)95                log_size                      = str(log_size).replace('\n','')96                print log_size97                ToGB                          = float(log_size)/(1024*1024*1024)98                ToGB                          = round(ToGB,2)99                print "current " + str(ToGB) + "GB"100                if yes_output_size_dict.has_key(dag):101                    yes_output_size_dict[dag] = yes_output_size_dict[dag] + ToGB102                    print str(dag) + " Total size :" + str(yes_output_size_dict[dag]) + "--second time"103                else:104                    yes_output_size_dict[dag] = ToGB105                    print str(dag) + " Total size : " + str(yes_ouput_size_dict[dag]) + "--first time"106    timestamp = str(time.strftime('%Y%m%d%H%M%S',time.localtime(time.time())));107    timestamp = ''108    file_obj_output                           = open("output_log_size_"+ timestamp+".txt",'w')109    yes_file_obj_output = open("yes_output_log_size_"+ timestamp +".txt",'w');110    file_obj_output_delta                     = open("delta_output_size_" + timestamp +".txt",'w')111    file_obj_output.write(day_time)112    file_obj_output.write('\n')113    114    for key in dag_list:115        file_obj_output.write(key)116        file_obj_output.write('\t')117        file_obj_output.write(str(output_size_dict[key]))118        file_obj_output.write('\n')119        file_obj_output_delta.write(key)120        file_obj_output_delta.write('\t')121        if yes_output_size_dict[key]!=0:122            huanbi = float(output_size_dict[key]-yes_output_size_dict[key])/yes_output_size_dict[key]123#            huanbi = huanbi*100.00124#            print ("%.2f" % huanbi)125#            huanbi = str(huanbi)+"%"126        else:127            huanbi = '0.0'128        delta_output_size_dict[key] = huanbi129        file_obj_output_delta.write(str(huanbi))130        file_obj_output_delta.write('\n')131        yes_file_obj_output.write(key);132        yes_file_obj_output.write('\t')133        yes_file_obj_output.write(str(yes_output_size_dict[key]))134        yes_file_obj_output.write('\n')135    136    file_obj_output.close()137    yes_file_obj_output.close()138    file_obj_output.close()139    file_obj_output_delta.close()140                        141def get_input_report(base_time):142    time_format = base_time.split('-')143    year        = time_format[0]144    month       = time_format[1]145    day         = time_format[2]146    147    mkdate = datetime.date(int(year),int(month), int(day))148    yesterday = str(mkdate - datetime.timedelta(days = 1))149    tomorrow  = str(mkdate + datetime.timedelta(days=1))150    yes_yes   = str(mkdate - datetime.timedelta(days = 2))151    152    yes       = yesterday.replace('-','')153    day_time  = base_time.replace('-','')154    yes_yes = yes_yes.replace('-','')155    print "yes_yes " + yes_yes156            157#get input log size158    for dag in dag_list:159        input_size_dict[dag]       = 0160        delta_input_size_dict[dag] = 0161        yes_input_size_dict[dag]   = 0162    for dag in dag_list:163        print "Current Dag is:" + str(dag)164        log_path       = dag_input_dict[dag]165        for log in log_path:166            print str(dag)+":"+str(log)167            ls_cmd     = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/" + day_time +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'168            yes_ls_cmd = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/" + yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'169            if dag=='udwetl_holmes4uap':170                ls_cmd = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/" + yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'171                yes_ls_cmd = HADOOP_CLIENT+" fs -ls "+log.strip()+ "/" + yes_yes +'>exit_file 2>&1;cat exit_file|grep "No such file or directory"'172                173            print "current yes command is :" + str(yes_ls_cmd)174            print "curren command is :" + str(ls_cmd)175            ls_cmd_re  = os.popen(ls_cmd).readlines()176            177            if ls_cmd_re:178                print "In "+dag+ " " + str(log)+" not exist "179                ls_cmd_re      = []180            else:181                print "In "+dag+ " " + str(log)+"  exist "182                ls_cmd_re      = []183                get_size_cmd = HADOOP_CLIENT + " fs -dus " + log.strip() + "/" + day_time + " | awk '{print $2}'"184                #handle holmes185                if dag == 'udwetl_holmes4uap':186                    get_size_cmd = HADOOP_CLIENT + " fs -dus " + log.strip() + "/" + yes + " | awk '{print $2}'"187                print get_size_cmd188                get_size_cmd_re          = os.popen(get_size_cmd).readlines()189                print type(get_size_cmd_re)190                log_size                 = str(get_size_cmd_re[0])191                print  type(log_size)192                log_size                 = str(log_size).replace('\n','')193                print log_size194                ToGB                     = float(log_size)/(1024*1024*1024)195                ToGB                     = round(ToGB,2)196                print str(ToGB) + "GB"197                if input_size_dict.has_key(dag):198                    input_size_dict[dag] = input_size_dict[dag] + ToGB199                    print str(dag) + "size :" + str(input_size_dict[dag]) + "--second time"200                else:201                    input_size_dict[dag] = ToGB202                    print str(dag) + " size : " + str(input_size_dict[dag]) + "--first time"203            yes_ls_cmd_re     = os.popen(yes_ls_cmd).readlines()204            if yes_ls_cmd_re:205                print "In "+dag+ " " + str(log)+" not exist "206                yes_ls_cmd_re = []207            else:208                print "In "+dag+ " " + str(log)+"  exist "209                yes_ls_cmd_re = []210                get_size_cmd  = HADOOP_CLIENT + " fs -dus " + log.strip() + "/" + yes + " | awk '{print $2}'"211                #handle holmes212                if dag == 'udwetl_holmes4uap':213                    get_size_cmd  = HADOOP_CLIENT + " fs -dus " + log.strip() + "/" + yes_yes + " | awk '{print $2}'"214                print get_size_cmd215                get_size_cmd_re              = os.popen(get_size_cmd).readlines()216                print type(get_size_cmd_re)217                log_size                     = str(get_size_cmd_re[0])218                print  type(log_size)219                log_size                     = str(log_size).replace('\n','')220                print log_size221                ToGB                         = float(log_size)/(1024*1024*1024)222                ToGB                         = round(ToGB,2)223                print str(ToGB) + "GB"224                if yes_input_size_dict.has_key(dag):225                    yes_input_size_dict[dag] = yes_input_size_dict[dag] + ToGB226                    print str(dag) + "size :" + str(yes_input_size_dict[dag]) + "--second time"227                else:228                    yes_input_size_dict[dag] = ToGB229                    print str(dag) + " size : " + str(yes_input_size_dict[dag]) + "--first time"230    file_obj_input                           = open("input_log_size.txt",'w')231    yes_file_obj_input = open("yes_input_log_size.txt",'w');232    file_obj_input_delta                     = open("delta_input_size.txt",'w')233    file_obj_input.write(day_time)234    file_obj_input.write('\n')235    236    for key in dag_list:237        file_obj_input.write(key)238        file_obj_input.write('\t')239        file_obj_input.write(str(input_size_dict[key]))240        file_obj_input.write('\n')241        file_obj_input_delta.write(key)242        file_obj_input_delta.write('\t')243        if yes_input_size_dict[key]!=0:244            huanbi = float(input_size_dict[key]-yes_input_size_dict[key])/yes_input_size_dict[key]245#            huanbi = huanbi*100.00246#            print ("%.2f" % huanbi)247#            huanbi = str(huanbi)+"%"248        else:249            huanbi = '0.0'250        delta_input_size_dict[key] =huanbi251        file_obj_input_delta.write(str(huanbi))252        file_obj_input_delta.write('\n')253        yes_file_obj_input.write(key);254        yes_file_obj_input.write('\t')255        yes_file_obj_input.write(str(yes_input_size_dict[key]))256        yes_file_obj_input.write('\n')257    258    file_obj_input.close()259    yes_file_obj_input.close()260    file_obj_input_delta.close()261    262if __name__=='__main__':263    init_structure()264    for dag in dag_list:265        get_slot_time(dag,sys.argv[1])266    print_dict()267    if len(sys.argv)==2:268        get_input_report(sys.argv[1])269        get_output_report(sys.argv[1])270    else:271        get_input_report('2013-08-26')272        get_output_report('2013-08-26')273    274    data = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]275    mailclient   = MailClient.MailClient()276    title = '[UDW Statistic] The Efficient Bill of UAP Mission [Mission Finish Time: '+str(sys.argv[1])+' ]'277    from_email  = 'zhangchao08@baidu.com'278    to_email    = 'zhangchao08@baidu.com'279    template_id = '24'280    for dag in dag_list:281        print '#------------------#'282        print input_size_dict[dag]283        print delta_input_size_dict[dag]284        print yes_output_size_dict[dag]285        print output_size_dict[dag]286        print delta_output_size_dict[dag]...

cmd.py

Source:cmd.py

1"""Wrappers for subprocess calls and associated helper functions."""2__author__ = "Alexander Gorelyshev"3__email__ = "alexander.gorelyshev@pm.me"4import asyncio5import logging6import os7import subprocess8import sys9from typing import Union10import magic  # type: ignore11from humanfriendly import format_size  # type: ignore12## ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ##13def get_cmd_output(cmd: str) -> dict:14    """Run a shell command via subprocess and return exit code and stdout/stderr.15    :param cmd: shell command as a string (supports redirects, wildcards, etc.)16    :returns: a dictionary with return code, stdout and stderr17    """18    proc = subprocess.run(19        cmd,20        shell=True,21        capture_output=True,22        text=True,23        check=False24    )25    return {"rc": proc.returncode, "stdout": proc.stdout.strip(), "stderr": proc.stderr.strip()}26## ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ##27async def get_cmd_output_async(cmd: str) -> dict:28    """Run an asynchronous shell command and return exit code and stdout/stderr.29    :param cmd: shell command as a string (supports redirects, wildcards, etc.)30    :returns: a dictionary with return code, stdout and stderr31    """32    proc = await asyncio.create_subprocess_shell(33        cmd,34        stdout=asyncio.subprocess.PIPE,35        stderr=asyncio.subprocess.PIPE,36        shell=True37    )38    stdout_b, stderr_b = await proc.communicate()39    stdout = stdout_b.decode("utf-8")40    stderr = stderr_b.decode("utf-8")41    return {"rc": proc.returncode, "stdout": stdout, "stderr": stderr}42## ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ##43def make_path_comment(path: str, message: str, logger: logging.LoggerAdapter, work_magic: bool = False, caller_funcname: bool = False) -> Union[str, None]:44    """Build a comment on a path, including size, other (optional) info and attach a message.45    :param path: path to be analyzed46    :param message: user-defined message, a report of some sort47    :param logger: logging.LoggerAdapter instance48    :param work_magic: (optional) add a python-magic description of the path [False]49    :param caller_funcname: (optional) get the name of the caller function [False]50    :returns: string comment containing requested bits of info plus the user-defined message51    """52    try:53        size_bytes = os.stat(path).st_size54        size = format_size(size_bytes)55        # size = f"{int(size_bytes / 1024)} kB" if size_bytes >= 1024 else f"{size_bytes} B"56        if work_magic:57            with open(path, "rb") as file:58                description = magic.from_buffer(file.read(2048))59        output = f"{message} ({size}, {description}): '{path}'"60        if caller_funcname:61            caller = sys._getframe(1).f_code.co_name62            output = f"({caller}) " + output if caller_funcname else output63    except Exception as error:64        logger.error(f"{repr(error)}: '{path}'")65        output = ""66    return output67## An alias for backwards-compatibility...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.