How to use session_log method in Slash

Best Python code snippet using slash

analyze.py

Source:analyze.py

...358        session_log.info('{setting}: {value} {gen_method}'.format(setting=setting_name, value=v, gen_method=auto))359    session_log.info('')360    _insert_section_separator(session_log)361    session_log.info('')362def _initialize_session_log(config):363    """Initial the log and dump the header ahead of analysis."""364    session_log = logging.getLogger(config.session_name)365    _write_session_log_header(config)366    session_log.info('BEGIN ANALYSIS')367    session_log.info('')368    return session_log369def _insert_analysis_summary_section(session_log):370    """Add the analysis summary section to the session log. Mainly for beautification."""371    session_log.info('')372    session_log.info('ANALYSIS COMPLETE')373    session_log.info('')374    _insert_section_separator(session_log)375    session_log.info('')376    session_log.info('SUMMARIES')377    session_log.info('')378def _save_parameters_and_skipped_wells(config, session_log, wells_table, results):379    """After the analysis is complete, examine the parameters and skipped well files380    (if any), and save them as TSVs and CSVs. The parameters file is large and381    contains all the columns corresponding to the `Parameters` object, although382    not all columns are populated. The `skipped` files are truncated versions of the383    input `wells_filename` as defined in the config."""384    parameter_fields = (385        'gene construct replicate well_file counts mean_fret gauss2_loc ' \386        'csat csat_slope linear_r2 max_gauss_r2 min_r2_region max_r2_region ' \387        'min_abs_diff_r2_region max_abs_diff_r2_region frac_above_csat color score'388    )389    fields = parameter_fields.split()390    # Prepare the `pandas.DataFrame` for use.391    parameters = OrderedDict()392    for field in fields:393        parameters[field] = list()394    # Determine which wells were skipped.395    skipped = list()396    for fargs in results:397        raw_well_file = fargs[1]398        params, _start_time, _stop_time = results[fargs]399        if params.counts == 0 and params.score == 0 and params.color == 'N/A':400            skipped.append(raw_well_file)401        for field in fields:402            col = getattr(params, field)403            parameters[field].append(col)404    skipped_dfs = list()405    for skipped_well in skipped:406        df = wells_table[wells_table['well_file'] == skipped_well].dropna()407        skipped_dfs.append(df)408    if len(skipped_dfs) > 0:409        _save_skipped_wells(config, session_log, skipped_dfs)410    _save_parameters(config, session_log, parameters)411    parameters_df = pd.DataFrame(parameters)412    return parameters_df, skipped413def _save_skipped_wells(config, session_log, skipped_dfs):414    """Export the skipped wells, and indicate as much in the session log."""415    skipped_df = pd.concat(skipped_dfs)416    skipped_df.reset_index(drop=True, inplace=True)417    skipped_savepath_tsv = os.path.join(config.work_dir, 'skipped.tsv')418    skipped_savepath_csv = os.path.join(config.work_dir, 'skipped.csv')419    to_fwf(skipped_df, skipped_savepath_tsv)420    skipped_df.to_csv(skipped_savepath_csv, index=False)421    session_log.info('Skipped well files exported as: "{}"'.format(skipped_savepath_tsv))422    session_log.info('Skipped well files exported as: "{}"'.format(skipped_savepath_csv))423    session_log.info('')424def _save_parameters(config, session_log, parameters):425    """Export the parameters, and indicate as much in the session log."""426    parameters_df           = pd.DataFrame(parameters)427    parameters_savepath_tsv = os.path.join(config.work_dir, 'parameters.tsv')428    parameters_savepath_csv = os.path.join(config.work_dir, 'parameters.csv')429    to_fwf(parameters_df, parameters_savepath_tsv)430    parameters_df.to_csv(parameters_savepath_csv, index=False)431    session_log.info('Classifications and scores exported as: "{}"'.format(parameters_savepath_tsv))432    session_log.info('Classifications and scores exported as: "{}"'.format(parameters_savepath_csv))433    session_log.info('')434def _log_classification_summary(logger, parameters_df, class_name):435    """Output the well files found with a given phase transition class."""436    if class_name.strip() == 'N/A':437        logger.info('Number of well files skipped: {}'.format(len(parameters_df)))438    else:439        logger.info('Number of well files identified with the class "{}": {}'.format(class_name, len(parameters_df)))440    logger.info('')441    for _index, row in parameters_df.iterrows():442        message = '{} : {}'.format(row['well_file'], row['score'])443        logger.info(message)444    logger.info('')445def _output_classifications_summary(logger, parameters_df):446    """Identify the unique 'colors' / 'classes' in the parameters `DataFrame` and output the447    wells that match that criterion."""448    colors = parameters_df['color'].to_numpy().tolist()449    unique_colors = list(sorted(set(colors)))450    logger.info('')451    logger.info('CLASSIFICATIONS SUMMARY')452    logger.info('')453    logger.info('<Well File> : <Confidence Score>')454    logger.info('')455    for color in unique_colors:456        df = parameters_df[parameters_df['color'] == color].dropna()457        _log_classification_summary(logger, df, color)458def _output_summary(session_log, wells_table, skipped, start_time):459    """Summarize and output the analysis performed."""460    total_wells         = len(wells_table['well_file'])461    skipped_wells       = len(skipped)462    analyzed_wells      = total_wells - skipped_wells463    stop_time           = datetime.now()464    processing_time     = stop_time - start_time465    session_log.info('SUMMARY OVERVIEW')466    session_log.info('')467    session_log.info('Number of wells to analyze:       {}'.format(total_wells))468    session_log.info('Number of wells analyzed:         {}'.format(analyzed_wells))469    session_log.info('Number of wells skipped:          {}'.format(skipped_wells))470    session_log.info('Processing time completed on:     {}'.format(stop_time))471    session_log.info('Total processing time:            {}'.format(processing_time))472    session_log.info('')473    _insert_section_separator(session_log)474def _move_plots_to_project_root(config, session_log):475    """Move all plots from the `work` directory to the `project_directory`."""476    session_log.info('')477    _insert_section_separator(session_log)478    session_log.info('')479    session_log.info('MOVE PLOTS')480    session_log.info('')481    session_log.info('Moving files from `work_directory` ("{}") to `project_directory` ({})'.format(482        config.work_dir,483        config.project_dir484    ))485    plot_files = [f for f in os.listdir(config.work_dir) if f.endswith(config.plot_type)]486    for fname in plot_files:487        original_path = os.path.join(config.work_dir, fname)488        path_for_copy = os.path.join(config.project_dir, fname)489        shutil.move(original_path, path_for_copy)490        session_log.info('Moved: "{}" to "{}"'.format(original_path, path_for_copy))491    session_log.info('')492    session_log.info('MOVE COMPLETE')493    session_log.info('')494    _insert_section_separator(session_log)495# ---------------------------------------------------------------------------------------------------------------------496# CLASSIFICATION FUNCTIONS497def classify_dataset(config, raw_well_name):498    """This function that does the primary work in classifying the dataset. It is specifically499    designed for multithreaded use in that each function call is indepedent of the other.500    @param config (Config):         A parsed and populated `Config` object specific to the501                                    project / dataset.502    @param raw_well_name (str):     The raw name of the well (i.e. zero-padded) which has503                                    to be parsed in other to determine the non-zero-padded504                                    filename that will be loaded and subsequently analyzed.505    @return params (Parameter):     A `Parameters` object which records what features were506                                    calculated.507    Once all the well files are analyzed in this way, the `Parameters` objects can then be508    collated and saved.509    """510    np.random.seed(config.random_seed)511    params = Parameters()512    logistic_func = lambda x, b, c: 1.0/(1.0+np.exp(-(x-b)/c))513    logistic_bounds = [[1, 0], [20, 2]]514    p0 = (5, 0.3)515    # Set the nice / priority level for the function when it516    # is executed. This becomes important when multiprocessing is applied.517    os.nice(config.nice_level)518    # We use the `raw_well_name` - e.g. A01 which has the leading zero.519    # The purpose of this is that when the log is saved, it can easily520    # be sorted and collated into the final session log.521    logger = _configure_logger(config.logs_dir, config.session_name, raw_well_name, attach_stream_handler=True)522    session = logging.getLogger(config.session_name)523    # Determine the actual well name524    well_name = determine_well_name(raw_well_name)525    well_file = os.path.join(config.project_dir, config.filename_format.format(well_name=well_name))526    # Begin processing.527    message = 'Processing well file "{}"'.format(well_file)528    session.info(message)529    logger.info(message)530    # First, check if this file exists531    if not os.path.exists(well_file):532        _log_skipped_file_not_found(session, well_file)533        _log_skipped_file_not_found(logger, well_file)534        params.well_file = raw_well_name535        return params536    # Next, check to determine if this file has enough cell measurements for analysis (as limited by the537    # Shannon entropy) due to the input grid size.538    raw_data, raw_counts = read_original_data(well_file, config.low_conc_cutoff, config.high_conc_cutoff, apply_cutoff=False)539    if raw_counts < config.min_measurements:540        _plot_skipped_well_file(config, raw_data, raw_well_name)541        _log_skipped_cell_measurements(session, well_file, config.min_measurements, raw_counts)542        _log_skipped_cell_measurements(logger, well_file, config.min_measurements, raw_counts)543        params.well_file = raw_well_name544        return params545    # Next, check if the file is within the data limits:546    valid, min_conc, max_conc = check_if_data_within_limits(raw_data, config.low_conc_cutoff, config.high_conc_cutoff)547    # But first, log the concentration limits548    _log_concentration_limits(session, well_file, config.low_conc_cutoff, config.high_conc_cutoff, min_conc, max_conc)549    _log_concentration_limits(logger, well_file, config.low_conc_cutoff, config.high_conc_cutoff, min_conc, max_conc)550    551    if not valid:552        _plot_skipped_well_file(config, raw_data, raw_well_name)553        _log_skipped_analysis(session, well_file)554        _log_skipped_analysis(logger, well_file)555        params.well_file = raw_well_name556        return params557    # At this point, we can continue558    data, counts        = apply_cutoff_to_dataframe(raw_data, config.low_conc_cutoff, config.high_conc_cutoff)559    params.counts       = counts560    params.well_file    = raw_well_name561    # remove extreme concentration values562    data = clamp_data(data, config.low_conc_cutoff, config.high_conc_cutoff)563    params.mean_fret = data['damfret'].mean()564    # plot the fine-grid plots565    if config.plot_fine_grids:566        plot_params = dict()567        plot_params['savepath']         = config.work_dir568        plot_params['well_name']        = raw_well_name569        plot_params['fg_conc_edges']    = config.fg_conc_edges570        plot_params['fg_fret_edges']    = config.fg_fret_edges571        plot_params['data_df']          = data572        plot_params['fine_grid_xlim']   = config.fine_grid_xlim573        plot_params['fine_grid_ylim']   = config.fine_grid_ylim574        plot_params['plot_type']        = config.plot_type575        plot_fine_grid_profiles(plot_params)576    # Calculate the concentration FRET averages and the max of these averages577    # This will then be used to characterize nucleation by setting the max average578    # as the center for a Gaussian, and setting another Gaussian at 0. A double579    # Gaussian function will be fit to the 1D histogram of the FRET for each concentration580    # slice, and the quality of that fit will serve as a proxy for nucleation.581    conc_fret_averages, slices_histograms = slice_data(data, config.conc_bins, config.fret_bins)582    average = max(conc_fret_averages[-4:])  # previously from [-4:]583    _log_conc_FRET_averages(logger, average, conc_fret_averages)584    # Determine how the system nucleates across concentration by calculating the area under the585    # Gaussian whose center is set to the max average. Then, that area is compared to the total586    # area under both Gaussians. Systems with specific nucleation profiles have unique587    # nucleation patterns.588    params.gauss2_loc = average589    nucleated_fractions, conc_bin_centers, r_squared = calculate_nucleated_fractions(config, raw_well_name, slices_histograms, average)590    # We also save the quality of fit of the double Gaussians.591    # Since the location of the second Gaussian (i.e. the max of the concentration averages) is592    # fixed, the quality of fit across the concentration slices is an additional proxy for593    # a continuous transition.594    conc_bin_centers    = np.array(conc_bin_centers)595    r_squared           = np.array(r_squared)596    params.max_gauss_r2 = np.max(r_squared)597    # fit a linear function to the R^2 values of the Gauss2 fits across the different slices.598    linear_func         = lambda x, a, b: a*x + b599    popt, pconv         = curve_fit(linear_func, conc_bin_centers, r_squared, bounds=[[-np.inf, -np.inf], [0, np.inf]])600    linear_func_data    = linear_func(conc_bin_centers, *popt)601    _p, linear_rsquared = calculate_rsquared(linear_func, conc_bin_centers, r_squared, bounds=[[-np.inf, -np.inf], [0, np.inf]])602    if config.plot_rsquared:603        plot_params = dict()604        plot_params['savepath']         = config.work_dir605        plot_params['conc_bin_centers'] = conc_bin_centers606        plot_params['r_squared']        = r_squared607        plot_params['linear_func_data'] = linear_func_data608        plot_params['well_name']        = raw_well_name609        plot_params['r']                = linear_rsquared610        plot_params['popt']             = popt611        plot_params['plot_type']        = config.plot_type612        plot_linear_rsquared_fit(plot_params)613    # now, analyze the fraction that's nucleated to determine if the system is in one or two-states614    diff = nucleated_fractions[-1] - np.min(nucleated_fractions)615    if nucleated_fractions[-1] - np.min(nucleated_fractions) < 0.15:616        mean_fret       = data['damfret'].mean()617        score           = (0.15-diff)/0.15618        params.score    = score619        if mean_fret < 0.05:620            params.color = 'blue'621        else:622            params.color = 'black'623        _log_class_and_score(session, well_file, params.color, params.score)624        _log_class_and_score(logger, well_file, params.color, params.score)625        return params626    # Now, fit a logistic function to the data627    popt, logistic_rs = calculate_rsquared(logistic_func, conc_bin_centers, nucleated_fractions, p0, logistic_bounds)628    saturation_conc, slope = popt629    logistic_y = logistic_func(conc_bin_centers, *popt)630    if config.plot_logistic:631        plot_params = dict()632        plot_params['savepath']             = config.work_dir633        plot_params['well_name']            = raw_well_name634        plot_params['conc_bin_centers']     = conc_bin_centers635        plot_params['nucleated_fractions']  = nucleated_fractions636        plot_params['logistic_y']           = logistic_y637        plot_params['r_squared']            = logistic_rs638        plot_params['popt']                 = popt639        plot_params['plot_type']            = config.plot_type640        plot_logistic_fits(plot_params)641    # This is the final check for black.642    if saturation_conc <= config.low_conc_cutoff:643        score = (0.15-diff)/0.15644        params.csat         = saturation_conc645        params.csat_slope   = slope646        params.color        = 'black'647        params.score        = 1.0648        _log_class_and_score(session, well_file, params.color, params.score)649        _log_class_and_score(logger, well_file, params.color, params.score)650        return params651    # Record the saturation concentration.652    logger.info('Saturation concentration: {}'.format(saturation_conc))653    if saturation_conc < max(conc_bin_centers):654        region_indices = np.where((conc_bin_centers >= saturation_conc - 1.0) & (conc_bin_centers <= saturation_conc + 1.0))655        logger.info('Saturation concentration type: lesser')656    else:657        indices = list(range(len(conc_bin_centers)))658        region_indices = indices[-5:]659        logger.info('Saturation concentration type: greater')660    region_lower = np.where(conc_bin_centers >= saturation_conc)661    region_upper = np.where(conc_bin_centers <= saturation_conc)662    _log_region_values(logger, region_indices, region_lower, region_upper, conc_bin_centers)663    # Determine if there is a large change in the fit goodness in this region. And, determine the max change in664    # fit goodness around the saturation concentration.665    region_r_squared = r_squared[region_indices]666    logger.info('Region R^2: {}'.format(region_r_squared))667    fit_value = np.max(np.abs(np.diff(region_r_squared)))668    params.csat                     = saturation_conc669    params.csat_slope               = slope670    params.linear_r2                = linear_rsquared671    params.min_r2_region            = np.min(region_r_squared)672    params.max_r2_region            = np.max(region_r_squared)673    params.min_abs_diff_r2_region   = np.min(np.abs(np.diff(region_r_squared)))674    params.max_abs_diff_r2_region   = np.max(np.abs(np.diff(region_r_squared)))675    # check for noisy data that hasn't fully phase separated676    upper_conc_limit = config.high_conc_cutoff - 1677    if saturation_conc <= upper_conc_limit:678        df = data[data['concentration'] > saturation_conc].dropna()679    else:680        df = data[data['concentration'] > upper_conc_limit].dropna()681    df = df[df['damfret'] > 0.05].dropna()682    total_points            = len(data)683    df_points               = len(df)684    fraction_above_csat     = df_points/total_points685    params.frac_above_csat  = fraction_above_csat686    # Finally, determine the phase separation class and its confidence score based on the687    # various parameters calculated.688    color, score = determine_class(df_points, fraction_above_csat, diff, fit_value, region_r_squared, linear_rsquared)689    params.color = color690    params.score = score691    # Log and quit.692    _log_class_and_score(session, well_file, color, score)693    _log_class_and_score(logger, well_file, color, score)694    return params695def classify_datasets(config_filename, move_to_project_root):696    """This is the main entry point for the classification of the datasets using697    a provided configuration file. This is so configured such that the user can698    use the entry point for any project directory structure.699    @param config_filename (str):       The location of the YAML config file which700                                        will be used when analyzing the data.701    @param move_to_project_root (bool): Whether or not to move the plots from702                                        the `work_dir` to the `project_directory`.703    @return None704    When this function completes, the analysis (`parameters.{tsv,csv}`) is saved705    to a `work` directory saved under the `project_directory` (if no `work` directory706    is provided). A session log is also output to that location, in addition to a707    `skipped.{tsv,csv}` file which is a truncated version of `wells_filename` that708    contains all the wells which were skipped during the analysis. All plots are709    either stored in the `work` directory by default, or in the `project_directory`710    if requested by setting `move_to_project_root` to `True` when calling this function.711    """712    start_time  = datetime.now()713    config      = Config(config_filename)714    # Setup the logger for use.715    iso_time                = start_time.strftime('%Y-%m-%d_%H-%M-%S')716    session_log_filename    = os.path.join(config.work_dir, 'session---%s.log' % iso_time)717    setup_logger(config.session_name, session_log_filename, attach_stream_handler=True)718    session_log = _initialize_session_log(config)719    wells_table = pd.read_csv(config.wells_filename)720    # Calculate the classification and score for each well.721    function_args = list()722    for _index, row in wells_table.iterrows():723        raw_well_name = str(row['well_file'])724        args = (config, raw_well_name, )725        function_args.append(args)726    # Save the results / parameters as an `OrderedDict`727    results = parallelize(classify_dataset, function_args, config.num_processes, config.session_name)728    if move_to_project_root:729        _move_plots_to_project_root(config, session_log)730    # Reformat and save the parameters and skipped wells (if any).731    _insert_analysis_summary_section(session_log)732    parameters, skipped = _save_parameters_and_skipped_wells(config, session_log, wells_table, results)...

SessionGlobalLogger.py

Source:SessionGlobalLogger.py

1import time2import copy3from settings import GlobalConfig4from session.SessionModels import (5    SessionSummary,6    SessionDeviceSummary,7    SessionTimeSummary,8    SessionApkSummary,9    SessionTestSummary,10    SessionFlakinessCheckSummary11)12from system.file import FileUtils13from system.console import (14    Printer,15    Color16)17import os18TAG = "SessionLogger:"19session_log = SessionSummary()20session_log.device_summaries = dict()21session_log.time_summary = SessionTimeSummary()22session_log.apk_summary = SessionApkSummary()23session_log.test_summary = SessionTestSummary()24session_log.test_summary.test_number = 025session_log.test_summary.test_passed = 026session_log.test_summary.test_failed = 027session_log.flakiness_summary = SessionFlakinessCheckSummary()28def _log_device(device_name):29    session_device_summary = session_log.device_summaries.get(device_name)30    if session_device_summary is None:31        session_device_summary = SessionDeviceSummary()32        session_device_summary.device_name = device_name33    session_log.device_summaries.update({device_name: session_device_summary})34def log_device_creation_start_time(device_name):35    _log_device(device_name)36    session_device_summary = session_log.device_summaries.get(device_name)37    if session_device_summary is None:38        session_device_summary = SessionDeviceSummary()39    if session_device_summary.creation_start_time is None:40        session_device_summary.creation_start_time = time.time()41    session_log.device_summaries.update({device_name: session_device_summary})42def log_device_creation_end_time(device_name):43    _log_device(device_name)44    session_device_summary = session_log.device_summaries.get(device_name)45    if session_device_summary is None:46        session_device_summary = SessionDeviceSummary()47    if session_device_summary.creation_end_time is None:48        session_device_summary.creation_end_time = time.time()49        _update_device_creation_time(session_device_summary)50    session_log.device_summaries.update({device_name: session_device_summary})51def _update_device_creation_time(session_device_summary):52    if session_device_summary.creation_start_time is not None and session_device_summary.creation_end_time is not None:53        session_device_summary.creation_time = \54            session_device_summary.creation_end_time - session_device_summary.creation_start_time55def log_device_launch_start_time(device_name):56    _log_device(device_name)57    session_device_summary = session_log.device_summaries.get(device_name)58    if session_device_summary is None:59        session_device_summary = SessionDeviceSummary()60    if session_device_summary.launch_start_time is None:61        session_device_summary.launch_start_time = time.time()62    session_log.device_summaries.update({device_name: session_device_summary})63def log_device_launch_end_time(device_name):64    _log_device(device_name)65    session_device_summary = session_log.device_summaries.get(device_name)66    if session_device_summary is None:67        session_device_summary = SessionDeviceSummary()68    if session_device_summary.launch_end_time is None:69        session_device_summary.launch_end_time = time.time()70        _update_device_launch_time(session_device_summary)71    session_log.device_summaries.update({device_name: session_device_summary})72def _update_device_launch_time(session_device_summary):73    if session_device_summary.launch_start_time is not None and session_device_summary.launch_end_time is not None:74        session_device_summary.launch_time = \75            session_device_summary.launch_end_time - session_device_summary.launch_start_time76def log_app_apk_install_start_time_on_device(device_name):77    _log_device(device_name)78    session_device_summary = session_log.device_summaries.get(device_name)79    if session_device_summary is None:80        session_device_summary = SessionDeviceSummary()81    if session_device_summary.apk_install_start_time is None:82        session_device_summary.apk_install_start_time = time.time()83    session_log.device_summaries.update({device_name: session_device_summary})84def log_app_apk_install_end_time_on_device(device_name):85    _log_device(device_name)86    session_device_summary = session_log.device_summaries.get(device_name)87    if session_device_summary is None:88        session_device_summary = SessionDeviceSummary()89    if session_device_summary.apk_install_end_time is None:90        session_device_summary.apk_install_end_time = time.time()91        _update_app_apk_install_time_on_device(session_device_summary)92    session_log.device_summaries.update({device_name: session_device_summary})93def _update_app_apk_install_time_on_device(session_device_summary):94    if session_device_summary.apk_install_start_time is not None and \95                    session_device_summary.apk_install_end_time is not None:96        session_device_summary.apk_install_time = \97            session_device_summary.apk_install_end_time - session_device_summary.apk_install_start_time98def log_test_apk_install_start_time_on_device(device_name):99    _log_device(device_name)100    session_device_summary = session_log.device_summaries.get(device_name)101    if session_device_summary is None:102        session_device_summary = SessionDeviceSummary()103    if session_device_summary.test_apk_install_start_time is None:104        session_device_summary.test_apk_install_start_time = time.time()105    session_log.device_summaries.update({device_name: session_device_summary})106def log_test_apk_install_end_time_on_device(device_name):107    _log_device(device_name)108    session_device_summary = session_log.device_summaries.get(device_name)109    if session_device_summary is None:110        session_device_summary = SessionDeviceSummary()111    if session_device_summary.test_apk_install_end_time is None:112        session_device_summary.test_apk_install_end_time = time.time()113        _update_test_apk_install_time_on_device(session_device_summary)114    session_log.device_summaries.update({device_name: session_device_summary})115def _update_test_apk_install_time_on_device(session_device_summary):116    if session_device_summary.test_apk_install_end_time is not None and \117                    session_device_summary.test_apk_install_start_time is not None:118        session_device_summary.test_apk_install_time = \119            session_device_summary.test_apk_install_end_time - session_device_summary.test_apk_install_start_time120def log_app_apk(app_apk_name):121    session_log.apk_summary.apk = app_apk_name122def log_test_apk(test_apk_name):123    session_log.apk_summary.test_apk = test_apk_name124def log_apk_version_code(version_code):125    session_log.apk_summary.version_code = version_code126def log_app_apk_build_start_time():127    session_log.apk_summary.apk_build_start_time = time.time()128def log_app_apk_build_end_time():129    session_log.apk_summary.apk_build_end_time = time.time()130    _update_app_apk_build_time()131def _update_app_apk_build_time():132    if session_log.apk_summary.apk_build_start_time is not None \133            and session_log.apk_summary.apk_build_end_time is not None:134        session_log.apk_summary.apk_build_time = session_log.apk_summary.apk_build_end_time - \135                                                 session_log.apk_summary.apk_build_start_time136def log_test_apk_build_start_time():137    if session_log.apk_summary.test_apk_build_start_time is None:138        session_log.apk_summary.test_apk_build_start_time = time.time()139def log_test_apk_build_end_time():140    if session_log.apk_summary.test_apk_build_end_time is None:141        session_log.apk_summary.test_apk_build_end_time = time.time()142        _update_test_apk_build_time()143def _update_test_apk_build_time():144    if session_log.apk_summary.test_apk_build_start_time is not None \145            and session_log.apk_summary.test_apk_build_end_time is not None:146        session_log.apk_summary.test_apk_build_time = session_log.apk_summary.test_apk_build_end_time - \147                                                      session_log.apk_summary.test_apk_build_start_time148def log_total_device_creation_start_time():149    if session_log.time_summary.total_device_creation_start_time is None:150        session_log.time_summary.total_device_creation_start_time = time.time()151def log_total_device_creation_end_time():152    if session_log.time_summary.total_device_creation_end_time is None:153        session_log.time_summary.total_device_creation_end_time = time.time()154        _update_total_device_creation_time()155def _update_total_device_creation_time():156    if session_log.time_summary.total_device_creation_start_time is not None \157            and session_log.time_summary.total_device_creation_end_time is not None:158        session_log.time_summary.total_device_creation_time = session_log.time_summary.total_device_creation_end_time \159                                                              - \160                                                              session_log.time_summary.total_device_creation_start_time161def log_total_device_launch_start_time():162    if session_log.time_summary.total_device_launch_start_time is None:163        session_log.time_summary.total_device_launch_start_time = time.time()164def log_total_device_launch_end_time():165    if session_log.time_summary.total_device_launch_end_time is None:166        session_log.time_summary.total_device_launch_end_time = time.time()167        _update_total_device_launch_time()168def _update_total_device_launch_time():169    if session_log.time_summary.total_device_launch_start_time is not None \170            and session_log.time_summary.total_device_launch_end_time is not None:171        session_log.time_summary.total_device_launch_time = session_log.time_summary.total_device_launch_end_time - \172                                                            session_log.time_summary.total_device_launch_start_time173def log_total_apk_build_start_time():174    if session_log.time_summary.total_apk_build_start_time is None:175        session_log.time_summary.total_apk_build_start_time = time.time()176def log_total_apk_build_end_time():177    if session_log.time_summary.total_apk_build_end_time is None:178        session_log.time_summary.total_apk_build_end_time = time.time()179        _update_total_apk_build_time()180def _update_total_apk_build_time():181    if session_log.time_summary.total_apk_build_start_time is not None \182            and session_log.time_summary.total_apk_build_end_time is not None:183        session_log.time_summary.total_apk_build_time = \184            session_log.time_summary.total_apk_build_end_time - session_log.time_summary.total_apk_build_start_time185def log_total_apk_install_start_time():186    if session_log.time_summary.total_apk_install_start_time is None:187        session_log.time_summary.total_apk_install_start_time = time.time()188def log_total_apk_install_end_time():189    if session_log.time_summary.total_apk_install_end_time is None:190        session_log.time_summary.total_apk_install_end_time = time.time()191        _update_total_apk_install_time()192def _update_total_apk_install_time():193    if session_log.time_summary.total_apk_install_start_time is not None \194            and session_log.time_summary.total_apk_install_end_time is not None:195        session_log.time_summary.total_apk_install_time = \196            session_log.time_summary.total_apk_install_end_time - session_log.time_summary.total_apk_install_start_time197def log_total_test_start_time():198    if session_log.time_summary.total_test_start_time is None:199        session_log.time_summary.total_test_start_time = time.time()200def log_total_test_end_time():201    if session_log.time_summary.total_test_end_time is None:202        session_log.time_summary.total_test_end_time = time.time()203        _update_total_test_time()204def _update_total_test_time():205    if session_log.time_summary.total_test_start_time is not None \206            and session_log.time_summary.total_test_end_time is not None:207        session_log.time_summary.total_test_time = \208            session_log.time_summary.total_test_end_time - session_log.time_summary.total_test_start_time209def log_total_rerun_start_time():210    if session_log.time_summary.total_rerun_start_time is None:211        session_log.time_summary.total_rerun_start_time = time.time()212def log_total_rerun_end_time():213    if session_log.time_summary.total_rerun_end_time is None:214        session_log.time_summary.total_rerun_end_time = time.time()215        _update_total_rerun_time()216def _update_total_rerun_time():217    if session_log.time_summary.total_rerun_start_time is not None \218            and session_log.time_summary.total_rerun_end_time is not None:219        session_log.time_summary.total_rerun_time = \220            session_log.time_summary.total_rerun_end_time - session_log.time_summary.total_rerun_start_time221def log_session_start_time():222    if session_log.time_summary.total_session_start_time is None:223        session_log.time_summary.total_session_start_time = time.time()224def log_session_end_time():225    if session_log.time_summary.total_session_end_time is None:226        session_log.time_summary.total_session_end_time = time.time()227        _update_total_session_time()228def _update_total_session_time():229    if session_log.time_summary.total_session_start_time is not None \230            and session_log.time_summary.total_session_end_time is not None:231        session_log.time_summary.total_session_time = \232            session_log.time_summary.total_session_end_time - session_log.time_summary.total_session_start_time233def increment_passed_tests():234    session_log.test_summary.test_passed += 1235    _update_test_number()236    _update_health_rate()237def increment_failed_tests():238    session_log.test_summary.test_failed += 1239    _update_test_number()240    _update_health_rate()241def _update_test_number():242    session_log.test_summary.test_number = session_log.test_summary.test_passed + session_log.test_summary.test_failed243def _update_health_rate():244    if session_log.test_summary.test_passed != 0 and session_log.test_summary.test_failed == 0:245        session_log.test_summary.health_rate = 1.00246    elif session_log.test_summary.test_passed == 0 and session_log.test_summary.test_failed != 0:247        session_log.test_summary.health_rate = 0.00248    elif session_log.test_summary.test_passed == 0 and session_log.test_summary.test_failed == 0:249        session_log.test_summary.health_rate = 1.00250    else:251        session_log.test_summary.health_rate = session_log.test_summary.test_passed / \252                                               session_log.test_summary.test_number253def update_flaky_candidate(test_summary):254    session_log.flakiness_summary.suspects.setdefault(test_summary.test_name, {255        "failed_count": 1,256        "passed_count": 0,257        "is_flaky": False258    })259    if test_summary.test_status == "success":260        session_log.flakiness_summary.suspects[test_summary.test_name]["passed_count"] += 1261        session_log.flakiness_summary.suspects[test_summary.test_name]["is_flaky"] = True262    else:263        session_log.flakiness_summary.suspects[test_summary.test_name]["failed_count"] += 1264def dump_session_summary():265    print_device_summaries()266    print_time_summary()267    print_apk_summary()268    print_test_summary()269    print_rerun_summary()270def print_device_summaries():271    if session_log.device_summaries:272        Printer.system_message(TAG, "Device details:")273        for key, device_summary in session_log.device_summaries.items():274            Printer.system_message(TAG, " - device " + Color.GREEN + device_summary.device_name + Color.BLUE + ":")275            if device_summary.creation_time is not None:276                Printer.system_message(TAG, "    * Creation time: " + Color.GREEN + "{:.2f}".format(277                    device_summary.creation_time) + Color.BLUE + " seconds.")278            if device_summary.launch_time is not None:279                Printer.system_message(TAG, "    * Launch time: " + Color.GREEN + "{:.2f}".format(280                    device_summary.launch_time) + Color.BLUE + " seconds.")281            if device_summary.apk_install_time is not None:282                Printer.system_message(TAG, "    * .*apk install time: " + Color.GREEN + "{:.2f}".format(283                    device_summary.apk_install_time) + Color.BLUE + " seconds.")284            if device_summary.test_apk_install_time is not None:285                Printer.system_message(TAG, "    * Test .*apk install time: " + Color.GREEN + "{:.2f}".format(286                    device_summary.test_apk_install_time) + Color.BLUE + " seconds.")287def print_time_summary():288    Printer.system_message(TAG, "Time details:")289    if session_log.time_summary.total_device_creation_time is not None:290        Printer.system_message(TAG, "  * Total device creation process took: " + Color.GREEN +291                               "{:.2f}".format(session_log.time_summary.total_device_creation_time) + Color.BLUE292                               + " seconds.")293    if session_log.time_summary.total_device_launch_time is not None:294        Printer.system_message(TAG, "  * Total device launch process took: " + Color.GREEN +295                               "{:.2f}".format(session_log.time_summary.total_device_launch_time) + Color.BLUE296                               + " seconds.")297    if session_log.time_summary.total_apk_build_time is not None:298        Printer.system_message(TAG, "  * Total .*apk build process took: " + Color.GREEN +299                               "{:.2f}".format(session_log.time_summary.total_apk_build_time) + Color.BLUE300                               + " seconds.")301    if session_log.time_summary.total_apk_install_time is not None:302        Printer.system_message(TAG, "  * Total .*apk installation process took: " + Color.GREEN +303                               "{:.2f}".format(session_log.time_summary.total_apk_install_time) + Color.BLUE304                               + " seconds.")305    if session_log.time_summary.total_test_time is not None:306        Printer.system_message(TAG, "  * Total test process took: " + Color.GREEN +307                               "{:.2f}".format(session_log.time_summary.total_test_time) + Color.BLUE + " seconds.")308    if session_log.time_summary.total_rerun_time is not None:309        Printer.system_message(TAG, "  * Total test re-run process took: " + Color.GREEN +310                               "{:.2f}".format(session_log.time_summary.total_rerun_time) + Color.BLUE + " seconds.")311    if session_log.time_summary.total_session_time is not None:312        Printer.system_message(TAG, "  * Total session time: " + Color.GREEN +313                               "{:.2f}".format(session_log.time_summary.total_session_time) + Color.BLUE + " seconds.")314def print_apk_summary():315    if session_log.apk_summary.apk is not None and session_log.apk_summary.apk_build_time is not None:316        Printer.system_message(TAG, ".*apk details:")317        Printer.system_message(TAG, "  * Application .*apk name: " + Color.GREEN + session_log.apk_summary.apk318                               + Color.BLUE + ".")319        if session_log.apk_summary.apk_build_time is not None:320            Printer.system_message(TAG, "  * Application .*apk build and scan time: " + Color.GREEN321                                   + "{:.2f}".format(session_log.apk_summary.apk_build_time) + Color.BLUE + " seconds.")322        if session_log.apk_summary.test_apk is not None:323            Printer.system_message(TAG, "  * Test .*apk name: " + Color.GREEN + session_log.apk_summary.test_apk324                                   + Color.BLUE + ".")325        if session_log.apk_summary.test_apk_build_time is not None:326            Printer.system_message(TAG, "  * Test .*apk build and scan time: " + Color.GREEN327                                   + "{:.2f}".format(session_log.apk_summary.test_apk_build_time) + Color.BLUE328                                   + " seconds.")329        if session_log.apk_summary.version_code is not None:330            Printer.system_message(TAG, "  * Version: " + Color.GREEN + str(session_log.apk_summary.version_code)331                                   + Color.BLUE + ".")332def print_test_summary():333    Printer.system_message(TAG, "Test details:")334    Printer.system_message(TAG, "  * Total number of test cases: " + Color.GREEN +335                           str(session_log.test_summary.test_number) + Color.BLUE + ".")336    Printer.system_message(TAG, "  * Tests passed: " + Color.GREEN + str(session_log.test_summary.test_passed)337                           + Color.BLUE + ".")338    Printer.system_message(TAG, "  * Tests failed: " + Color.GREEN + str(session_log.test_summary.test_failed)339                           + Color.BLUE + ".")340    if session_log.test_summary.health_rate is not None:341        Printer.system_message(TAG, "  * Health rate: " + Color.GREEN342                               + "{0:.2f}%".format(session_log.test_summary.health_rate * 100) + Color.BLUE + ".")343def print_rerun_summary():344    if GlobalConfig.SHOULD_RERUN_FAILED_TESTS:345        Printer.system_message(TAG, "Re-run details:")346        Printer.system_message(TAG, "  * from " + Color.GREEN + "{}".format(session_log.test_summary.test_failed)347                               + Color.BLUE + " failed test cases, each was started again " + Color.GREEN348                               + "{}".format(GlobalConfig.FLAKINESS_RERUN_COUNT) + Color.BLUE + " times:")349        for suspect, status in session_log.flakiness_summary.suspects.items():350            Printer.system_message(TAG, "   * (failed: " + Color.GREEN +351                                   "{}".format(status["failed_count"]) + Color.BLUE + ", passed: " + Color.GREEN352                                   + "{}".format(status["passed_count"]) + Color.BLUE + ", is_flaky: " + Color.GREEN353                                   + "{}".format(status["is_flaky"]) + Color.BLUE + ") {}".format(suspect))354def dump_saved_files_history():355    nothing_to_display = True356    if FileUtils.dir_exists(GlobalConfig.OUTPUT_AVD_LOG_DIR):357        nothing_to_display = False358        saved_avd_logs = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_AVD_LOG_DIR)359        Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_AVD_LOG_DIR)360                               + Color.BLUE + " (" + Color.GREEN + str(len(saved_avd_logs)) + " files" + Color.BLUE361                               + ")")362        for saved_file in FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_AVD_LOG_DIR):363            Printer.system_message(TAG, "  * " + saved_file + ".")364    if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_LOG_DIR):365        nothing_to_display = False366        saved_test_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOG_DIR)367        Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_LOG_DIR)368                               + Color.BLUE + " (" + Color.GREEN + str(len(saved_test_summaries)) + " files"369                               + Color.BLUE + ")")370        for saved_file in FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOG_DIR):371            Printer.system_message(TAG, "  * " + saved_file + ".")372    if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR):373        nothing_to_display = False374        saved_logcats_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR)375        Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR)376                               + Color.BLUE + " (" + Color.GREEN + str(len(saved_logcats_summaries)) + " files"377                               + Color.BLUE + ")")378        for saved_file in saved_logcats_summaries:379            Printer.system_message(TAG, "  * " + saved_file + ".")380    if GlobalConfig.SHOULD_RECORD_TESTS:381        if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR):382            nothing_to_display = False383            saved_recordings_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR)384            Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR)385                                   + Color.BLUE + " (" + Color.GREEN + str(len(saved_recordings_summaries))386                                   + " files" + Color.BLUE + ")")387            for saved_file in saved_recordings_summaries:388                Printer.system_message(TAG, "  * " + saved_file + ".")389    if nothing_to_display:390        Printer.system_message(TAG, "No files were saved during session.")391def save_session_summary():392    if GlobalConfig.OUTPUT_SUMMARY_LOG_DIR is not None and os.path.exists(GlobalConfig.OUTPUT_SUMMARY_LOG_DIR):393        session_log_json_dict = copy.deepcopy(session_log)394        session_log_json_dict.time_summary = vars(session_log.time_summary)395        session_log_json_dict.apk_summary = vars(session_log.apk_summary)396        session_log_json_dict.test_summary = vars(session_log.test_summary)397        session_log_json_dict.device_summaries = list()398        for device_name, device_summary in session_log.device_summaries.items():399            session_log_json_dict.device_summaries.append(vars(device_summary))400        session_log_json_dict.flakiness_summary = vars(session_log.flakiness_summary)401        session_log_json_dict = vars(session_log_json_dict)...

irc_extractor.py

Source:irc_extractor.py

1import json2import os3import socket4from collections import defaultdict5from copy import deepcopy6import dpkt7import numpy as np8import re9import irc_config10def compute_session_periodicity(communication):11    """12    using Fast Fourier Transform to compute periodicity of messages in sessions13    @:return value in [0,1] interval - 0 means that messages are not periodic at all14    """15    if len(communication) < 3:16        return None17    t = list(map(lambda x: x['timestamp'], communication))18    td = np.asarray(list(map(lambda x: x[0] - x[1], zip(t[1:], t))))19    fft_res = np.absolute(np.fft.fft(td))20    T = fft_res.argmax() + 221    rng_size = int(len(td) / T)22    td_T = [td[x * T:x * T + T] for x in range(rng_size)]23    td_T_avg = np.mean(td_T, 0)24    # ||td_t - td_avg ||2 / ||td_t||25    td_nmse = np.linalg.norm(td_T - td_T_avg) / np.linalg.norm(td_T)26    return 1 - td_nmse27def compute_msg_periodicity(communication):28    if len(communication) < 3:29        return len(communication) * [None]30    t = list(map(lambda x: x['timestamp'], communication))31    msg_per = list()32    # not able to compute last element periodicity - no successor msg33    msg_per.append(None)34    for ta, tb, tc in zip(t, t[1:], t[2:]):35        t1 = tb - ta36        t2 = tc - tb37        try:38            msg_per.append(t2 / t1)39        except ZeroDivisionError:40            msg_per.append(None)41            print('t1 = {} - {} =  {}, t2:{} - {} = {},'.format(tb, ta, t1, tc, tb, t2))42    # not able to compute last element periodicity - no successor msg43    msg_per.append(None)44    return msg_per45class IRCExtractor:46    irc_port_dict = {'2.irc': 2407, '3.irc': 2407, '4.irc': 6667, '34.irc': 2407, '39.irc': 6667, '42.irc': 4975,47                     '51.irc': 54468, '56.irc': 80, '62.irc': 443, 'irc1': 6667, 'irc3': 6667}48    def __init__(self, pcap):49        self.irc_logs = defaultdict(lambda: [])50        self.irc_packet_counter = 051        self.unfinished_msg = ''52        self.msg_not_finished = False53        self.irc_port = self.irc_port_dict[pcap]54    def process_packet(self, timestamp, buffer):55        eth = dpkt.ethernet.Ethernet(buffer)56        if eth.type != dpkt.ethernet.ETH_TYPE_IP and eth.type != dpkt.ethernet.ETH_TYPE_8021Q:57            # not ip packet58            return59        try:60            ip = eth.data61            # not tcp packet62            if type(ip.data) != dpkt.tcp.TCP:63                return64        except AttributeError:65            return 66            67        try:68            tcp = ip.data69            ip_src = socket.inet_ntoa(ip.src)70            ip_dst = socket.inet_ntoa(ip.dst)71            sport = tcp.sport72            dport = tcp.dport73            data = tcp.data74        except OSError:75            return76        if sport != self.irc_port and dport != self.irc_port:77            # invalid port78            return79        if self.irc_packet_counter % 1000 == 0:80            print('IRC PRIVMSG Packet #{}'.format(self.irc_packet_counter))81        try:82            payload = data.decode('ascii')83        except:84            return85        self.irc_packet_counter += 186        if data and 'PRIVMSG' in payload:87            # payload = data.decode('utf8')88            # payload schema is (list of) src PRIVMSG dst msg \r\n, so we split them by \r\n89            p_splitted = payload.split('\r\n')90            p_len = len(p_splitted)91            for i, msg in enumerate(p_splitted):92                if i == 0 and self.msg_not_finished:93                    self.msg_not_finished = False94                    msg = self.unfinished_msg + msg95                if i == p_len - 1 and payload[:4] != '\r\n':96                    self.msg_not_finished = True97                    self.unfinished_msg = msg98                    return99                msg_splitted = msg.split()100                if len(msg_splitted) < 4 or msg_splitted[1] != 'PRIVMSG':101                    continue102                # msg starts with :, so it can be neglected103                src, dst, msg_text = msg_splitted[0], msg_splitted[2], ' '.join(msg_splitted[3:])[1:]104                irc_log = {'timestamp': timestamp, 'msg': msg_text, 'pkt_size': len(buffer), 'sport': sport}105                self.irc_logs[((src, ip_src), (dst, ip_dst, dport))].append(irc_log)106    def save_logs(self, pcap_out_json_path, malicious):107        logs = []108        for session, communication in self.irc_logs.items():109            session_log = {}110            src, dst = session[0], session[1]111            msg_times = list(map(lambda c: c['timestamp'], communication))112            pkt_sizes = list(map(lambda c: c['pkt_size'], communication))113            session_log['src'] = src[0]114            session_log['src_ip'] = src[1]115            session_log['src_ports'] = list(set(map(lambda x: x['sport'], communication)))116            for msg in communication:117                del msg['sport']118            119            session_log['dst'] = dst[0]120            session_log['dst_ip'] = dst[1]121            session_log['dst_port'] = dst[2]122            session_log['start_time'] = min(msg_times)123            session_log['end_time'] = max(msg_times)124            session_log['duration'] =  session_log['end_time'] - session_log['start_time']125            session_log['msg_count'] = len(communication)126            session_log['pkt_size_total'] = sum(pkt_sizes)127            per = compute_session_periodicity(communication)128            if per is not None:129                session_log['periodicity'] = per130            else:131                session_log['periodicity'] = np.nan132            msg_per = compute_msg_periodicity(communication)133            134            # MODEL 2135            src = src[0]136            usr_rgx = re.match(r'^.*?(?=@)', src + '@')137            src_username = src[: usr_rgx.regs[0][1]]138            spec_chars_username_mean = 0 if len(src_username) == 0 else len(re.findall(r'[^A-Za-z]', src_username)) / len(src_username)139            msg_special_chars = []140            msg_word_count = defaultdict(lambda: 0)141            comm2 = []142            for msg, p in zip(communication, msg_per):143                msg['periodicity'] = p if p is not None else np.nan144                msg_content = msg['msg']145                msg_words = msg_content.split()146                for word in msg_words:147                    msg_word_count[word] += 1148                msg_spec = 0 if len(msg_content) == 0 else len(re.findall(r'[^A-Za-z]', msg_content)) / len(msg_content)149                msg_special_chars.append(msg_spec)150                151                comm2.append(msg)152            _wordcounts = list(msg_word_count.values())153            p = _wordcounts / np.sum(_wordcounts)154            msg_word_entropy = -np.sum(p * np.log2(p))155            spec_chars_msg_mean = np.mean(msg_special_chars)156            session_log['spec_chars_username_mean'] = spec_chars_username_mean157            session_log['spec_chars_msg_mean'] = spec_chars_msg_mean158            session_log['msg_word_entropy'] = msg_word_entropy159            session_log['malicious'] = malicious160            session_log['msgs'] = comm2161            logs.append(session_log)162        final_irc_logs = {'sessions': logs}163        self.irc_logs = logs164        with open(pcap_out_json_path, 'w+', encoding='utf-8') as f:165            f.write(json.dumps(final_irc_logs, indent=4))166    def filter_logs(self, irc_logs, pcap_json_path):167        logs = []168        for session, communication in irc_logs.items():169            filtered_com = []170            for c in communication:171                if c['src'] != 'PRIVMSG' and c['src'] != 'NOTICE' and c['dst'] != 'PRIVMSG' and c['dst'] != 'NOTICE':172                    filtered_com.append(c)173            if len(filtered_com) > 0:174                logs.append({session: filtered_com})175        final_irc_logs = {'data': logs}176        with open(pcap_json_path, 'w+', encoding='utf-8') as f:177            f.write(json.dumps(final_irc_logs, indent=4))178    def load_logs(self, json_filename):179        print('loading logs...')180        self.irc_logs = defaultdict(lambda: [])181        with open(json_filename, 'r') as f:182            self.irc_logs = json.load(f)['sessions']183        return deepcopy(self.irc_logs)184    def sniff_pcap(self, pcap_path, pcap_out_json_path, malicious):185        print('sniffing pcap...')186        self.irc_logs = defaultdict(lambda: [])187        with open(pcap_path, 'rb') as f:188            pcap = dpkt.pcap.Reader(f)189            for ts, buf in pcap:190                # checking for zero ethertype and throwing away:191                if len(buf) > 0:192                    try:193                        self.process_packet(ts, buf)194                    except:195                        continue196 197        self.save_logs(pcap_out_json_path, malicious)198        return self.irc_logs199def main():200    for pcap in irc_config.PCAPS.items():201        print('Sniffing PCAP {}...'.format(pcap[0]))202        irc_extractor = IRCExtractor(pcap[0])203        if os.path.isfile(irc_config.pcap_json_path(pcap[0])):204            print(irc_config.pcap_json_path(pcap[0]))205            irc_logs = irc_extractor.load_logs(irc_config.pcap_json_path(pcap[0]))206            # filter_logs()207        else:208            irc_logs = irc_extractor.sniff_pcap(irc_config.pcap_path(pcap), irc_config.pcap_json_path(pcap[0]), malicious=pcap[1])209    # irc_extractor.extract_features()210    # irc_logs.sort(key=lambda x: x.get('timestamp'))211    # graph = build_graph(irc_logs)212    # visualize_graph(graph, PCAP_GRAPH_PATH)213if __name__ == '__main__':...

test_netmiko_sesssion_log.py

Source:test_netmiko_sesssion_log.py

...14            contents = f.read()15    else:16        raise ValueError("Most specify either file_name or contents")17    return hashlib.md5(contents).hexdigest()18def read_session_log(session_file, append=False):19    """Leading white-space can vary. Strip off leading white-space."""20    with open(session_file, "rb") as f:21        if append is True:22            line = f.readline().decode()23            assert "Initial file contents" in line24        log_content = f.read().lstrip()25        return log_content26def session_action(my_connect, command):27    """Common actions in the netmiko session to generate the session log."""28    time.sleep(1)29    my_connect.clear_buffer()30    output = my_connect.send_command(command)31    my_connect.disconnect()32    return output33def session_log_md5(session_file, compare_file):34    """Compare the session_log MD5 to the compare_file MD5"""35    compare_log_md5 = calc_md5(file_name=compare_file)36    log_content = read_session_log(session_file)37    session_log_md5 = calc_md5(contents=log_content)38    assert session_log_md5 == compare_log_md539def session_log_md5_append(session_file, compare_file):40    """Compare the session_log MD5 to the compare_file MD5"""41    compare_log_md5 = calc_md5(file_name=compare_file)42    log_content = read_session_log(session_file, append=True)43    session_log_md5 = calc_md5(contents=log_content)44    assert session_log_md5 == compare_log_md545def test_session_log(net_connect, commands, expected_responses):46    """Verify session_log matches expected content."""47    command = commands["basic"]48    session_action(net_connect, command)49    compare_file = expected_responses["compare_log"]50    session_file = expected_responses["session_log"]51    session_log_md5(session_file, compare_file)52def test_session_log_write(net_connect_slog_wr, commands, expected_responses):53    """Verify session_log matches expected content, but when channel writes are also logged."""54    command = commands["basic"]55    session_action(net_connect_slog_wr, command)56    compare_file = expected_responses["compare_log_wr"]57    session_file = expected_responses["session_log"]58    session_log_md5(session_file, compare_file)59def test_session_log_append(device_slog, commands, expected_responses):...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.