How to use session_log method in Slash

Best Python code snippet using slash

analyze.py

Source:analyze.py Github

copy

Full Screen

...358 session_log.info('{setting}: {value} {gen_method}'.format(setting=setting_name, value=v, gen_method=auto))359 session_log.info('')360 _insert_section_separator(session_log)361 session_log.info('')362def _initialize_session_log(config):363 """Initial the log and dump the header ahead of analysis."""364 session_log = logging.getLogger(config.session_name)365 _write_session_log_header(config)366 session_log.info('BEGIN ANALYSIS')367 session_log.info('')368 return session_log369def _insert_analysis_summary_section(session_log):370 """Add the analysis summary section to the session log. Mainly for beautification."""371 session_log.info('')372 session_log.info('ANALYSIS COMPLETE')373 session_log.info('')374 _insert_section_separator(session_log)375 session_log.info('')376 session_log.info('SUMMARIES')377 session_log.info('')378def _save_parameters_and_skipped_wells(config, session_log, wells_table, results):379 """After the analysis is complete, examine the parameters and skipped well files380 (if any), and save them as TSVs and CSVs. The parameters file is large and381 contains all the columns corresponding to the `Parameters` object, although382 not all columns are populated. The `skipped` files are truncated versions of the383 input `wells_filename` as defined in the config."""384 parameter_fields = (385 'gene construct replicate well_file counts mean_fret gauss2_loc ' \386 'csat csat_slope linear_r2 max_gauss_r2 min_r2_region max_r2_region ' \387 'min_abs_diff_r2_region max_abs_diff_r2_region frac_above_csat color score'388 )389 fields = parameter_fields.split()390 # Prepare the `pandas.DataFrame` for use.391 parameters = OrderedDict()392 for field in fields:393 parameters[field] = list()394 # Determine which wells were skipped.395 skipped = list()396 for fargs in results:397 raw_well_file = fargs[1]398 params, _start_time, _stop_time = results[fargs]399 if params.counts == 0 and params.score == 0 and params.color == 'N/​A':400 skipped.append(raw_well_file)401 for field in fields:402 col = getattr(params, field)403 parameters[field].append(col)404 skipped_dfs = list()405 for skipped_well in skipped:406 df = wells_table[wells_table['well_file'] == skipped_well].dropna()407 skipped_dfs.append(df)408 if len(skipped_dfs) > 0:409 _save_skipped_wells(config, session_log, skipped_dfs)410 _save_parameters(config, session_log, parameters)411 parameters_df = pd.DataFrame(parameters)412 return parameters_df, skipped413def _save_skipped_wells(config, session_log, skipped_dfs):414 """Export the skipped wells, and indicate as much in the session log."""415 skipped_df = pd.concat(skipped_dfs)416 skipped_df.reset_index(drop=True, inplace=True)417 skipped_savepath_tsv = os.path.join(config.work_dir, 'skipped.tsv')418 skipped_savepath_csv = os.path.join(config.work_dir, 'skipped.csv')419 to_fwf(skipped_df, skipped_savepath_tsv)420 skipped_df.to_csv(skipped_savepath_csv, index=False)421 session_log.info('Skipped well files exported as: "{}"'.format(skipped_savepath_tsv))422 session_log.info('Skipped well files exported as: "{}"'.format(skipped_savepath_csv))423 session_log.info('')424def _save_parameters(config, session_log, parameters):425 """Export the parameters, and indicate as much in the session log."""426 parameters_df = pd.DataFrame(parameters)427 parameters_savepath_tsv = os.path.join(config.work_dir, 'parameters.tsv')428 parameters_savepath_csv = os.path.join(config.work_dir, 'parameters.csv')429 to_fwf(parameters_df, parameters_savepath_tsv)430 parameters_df.to_csv(parameters_savepath_csv, index=False)431 session_log.info('Classifications and scores exported as: "{}"'.format(parameters_savepath_tsv))432 session_log.info('Classifications and scores exported as: "{}"'.format(parameters_savepath_csv))433 session_log.info('')434def _log_classification_summary(logger, parameters_df, class_name):435 """Output the well files found with a given phase transition class."""436 if class_name.strip() == 'N/​A':437 logger.info('Number of well files skipped: {}'.format(len(parameters_df)))438 else:439 logger.info('Number of well files identified with the class "{}": {}'.format(class_name, len(parameters_df)))440 logger.info('')441 for _index, row in parameters_df.iterrows():442 message = '{} : {}'.format(row['well_file'], row['score'])443 logger.info(message)444 logger.info('')445def _output_classifications_summary(logger, parameters_df):446 """Identify the unique 'colors' /​ 'classes' in the parameters `DataFrame` and output the447 wells that match that criterion."""448 colors = parameters_df['color'].to_numpy().tolist()449 unique_colors = list(sorted(set(colors)))450 logger.info('')451 logger.info('CLASSIFICATIONS SUMMARY')452 logger.info('')453 logger.info('<Well File> : <Confidence Score>')454 logger.info('')455 for color in unique_colors:456 df = parameters_df[parameters_df['color'] == color].dropna()457 _log_classification_summary(logger, df, color)458def _output_summary(session_log, wells_table, skipped, start_time):459 """Summarize and output the analysis performed."""460 total_wells = len(wells_table['well_file'])461 skipped_wells = len(skipped)462 analyzed_wells = total_wells - skipped_wells463 stop_time = datetime.now()464 processing_time = stop_time - start_time465 session_log.info('SUMMARY OVERVIEW')466 session_log.info('')467 session_log.info('Number of wells to analyze: {}'.format(total_wells))468 session_log.info('Number of wells analyzed: {}'.format(analyzed_wells))469 session_log.info('Number of wells skipped: {}'.format(skipped_wells))470 session_log.info('Processing time completed on: {}'.format(stop_time))471 session_log.info('Total processing time: {}'.format(processing_time))472 session_log.info('')473 _insert_section_separator(session_log)474def _move_plots_to_project_root(config, session_log):475 """Move all plots from the `work` directory to the `project_directory`."""476 session_log.info('')477 _insert_section_separator(session_log)478 session_log.info('')479 session_log.info('MOVE PLOTS')480 session_log.info('')481 session_log.info('Moving files from `work_directory` ("{}") to `project_directory` ({})'.format(482 config.work_dir,483 config.project_dir484 ))485 plot_files = [f for f in os.listdir(config.work_dir) if f.endswith(config.plot_type)]486 for fname in plot_files:487 original_path = os.path.join(config.work_dir, fname)488 path_for_copy = os.path.join(config.project_dir, fname)489 shutil.move(original_path, path_for_copy)490 session_log.info('Moved: "{}" to "{}"'.format(original_path, path_for_copy))491 session_log.info('')492 session_log.info('MOVE COMPLETE')493 session_log.info('')494 _insert_section_separator(session_log)495# ---------------------------------------------------------------------------------------------------------------------496# CLASSIFICATION FUNCTIONS497def classify_dataset(config, raw_well_name):498 """This function that does the primary work in classifying the dataset. It is specifically499 designed for multithreaded use in that each function call is indepedent of the other.500 @param config (Config): A parsed and populated `Config` object specific to the501 project /​ dataset.502 @param raw_well_name (str): The raw name of the well (i.e. zero-padded) which has503 to be parsed in other to determine the non-zero-padded504 filename that will be loaded and subsequently analyzed.505 @return params (Parameter): A `Parameters` object which records what features were506 calculated.507 Once all the well files are analyzed in this way, the `Parameters` objects can then be508 collated and saved.509 """510 np.random.seed(config.random_seed)511 params = Parameters()512 logistic_func = lambda x, b, c: 1.0/​(1.0+np.exp(-(x-b)/​c))513 logistic_bounds = [[1, 0], [20, 2]]514 p0 = (5, 0.3)515 # Set the nice /​ priority level for the function when it516 # is executed. This becomes important when multiprocessing is applied.517 os.nice(config.nice_level)518 # We use the `raw_well_name` - e.g. A01 which has the leading zero.519 # The purpose of this is that when the log is saved, it can easily520 # be sorted and collated into the final session log.521 logger = _configure_logger(config.logs_dir, config.session_name, raw_well_name, attach_stream_handler=True)522 session = logging.getLogger(config.session_name)523 # Determine the actual well name524 well_name = determine_well_name(raw_well_name)525 well_file = os.path.join(config.project_dir, config.filename_format.format(well_name=well_name))526 # Begin processing.527 message = 'Processing well file "{}"'.format(well_file)528 session.info(message)529 logger.info(message)530 # First, check if this file exists531 if not os.path.exists(well_file):532 _log_skipped_file_not_found(session, well_file)533 _log_skipped_file_not_found(logger, well_file)534 params.well_file = raw_well_name535 return params536 # Next, check to determine if this file has enough cell measurements for analysis (as limited by the537 # Shannon entropy) due to the input grid size.538 raw_data, raw_counts = read_original_data(well_file, config.low_conc_cutoff, config.high_conc_cutoff, apply_cutoff=False)539 if raw_counts < config.min_measurements:540 _plot_skipped_well_file(config, raw_data, raw_well_name)541 _log_skipped_cell_measurements(session, well_file, config.min_measurements, raw_counts)542 _log_skipped_cell_measurements(logger, well_file, config.min_measurements, raw_counts)543 params.well_file = raw_well_name544 return params545 # Next, check if the file is within the data limits:546 valid, min_conc, max_conc = check_if_data_within_limits(raw_data, config.low_conc_cutoff, config.high_conc_cutoff)547 # But first, log the concentration limits548 _log_concentration_limits(session, well_file, config.low_conc_cutoff, config.high_conc_cutoff, min_conc, max_conc)549 _log_concentration_limits(logger, well_file, config.low_conc_cutoff, config.high_conc_cutoff, min_conc, max_conc)550 551 if not valid:552 _plot_skipped_well_file(config, raw_data, raw_well_name)553 _log_skipped_analysis(session, well_file)554 _log_skipped_analysis(logger, well_file)555 params.well_file = raw_well_name556 return params557 # At this point, we can continue558 data, counts = apply_cutoff_to_dataframe(raw_data, config.low_conc_cutoff, config.high_conc_cutoff)559 params.counts = counts560 params.well_file = raw_well_name561 # remove extreme concentration values562 data = clamp_data(data, config.low_conc_cutoff, config.high_conc_cutoff)563 params.mean_fret = data['damfret'].mean()564 # plot the fine-grid plots565 if config.plot_fine_grids:566 plot_params = dict()567 plot_params['savepath'] = config.work_dir568 plot_params['well_name'] = raw_well_name569 plot_params['fg_conc_edges'] = config.fg_conc_edges570 plot_params['fg_fret_edges'] = config.fg_fret_edges571 plot_params['data_df'] = data572 plot_params['fine_grid_xlim'] = config.fine_grid_xlim573 plot_params['fine_grid_ylim'] = config.fine_grid_ylim574 plot_params['plot_type'] = config.plot_type575 plot_fine_grid_profiles(plot_params)576 # Calculate the concentration FRET averages and the max of these averages577 # This will then be used to characterize nucleation by setting the max average578 # as the center for a Gaussian, and setting another Gaussian at 0. A double579 # Gaussian function will be fit to the 1D histogram of the FRET for each concentration580 # slice, and the quality of that fit will serve as a proxy for nucleation.581 conc_fret_averages, slices_histograms = slice_data(data, config.conc_bins, config.fret_bins)582 average = max(conc_fret_averages[-4:]) # previously from [-4:]583 _log_conc_FRET_averages(logger, average, conc_fret_averages)584 # Determine how the system nucleates across concentration by calculating the area under the585 # Gaussian whose center is set to the max average. Then, that area is compared to the total586 # area under both Gaussians. Systems with specific nucleation profiles have unique587 # nucleation patterns.588 params.gauss2_loc = average589 nucleated_fractions, conc_bin_centers, r_squared = calculate_nucleated_fractions(config, raw_well_name, slices_histograms, average)590 # We also save the quality of fit of the double Gaussians.591 # Since the location of the second Gaussian (i.e. the max of the concentration averages) is592 # fixed, the quality of fit across the concentration slices is an additional proxy for593 # a continuous transition.594 conc_bin_centers = np.array(conc_bin_centers)595 r_squared = np.array(r_squared)596 params.max_gauss_r2 = np.max(r_squared)597 # fit a linear function to the R^2 values of the Gauss2 fits across the different slices.598 linear_func = lambda x, a, b: a*x + b599 popt, pconv = curve_fit(linear_func, conc_bin_centers, r_squared, bounds=[[-np.inf, -np.inf], [0, np.inf]])600 linear_func_data = linear_func(conc_bin_centers, *popt)601 _p, linear_rsquared = calculate_rsquared(linear_func, conc_bin_centers, r_squared, bounds=[[-np.inf, -np.inf], [0, np.inf]])602 if config.plot_rsquared:603 plot_params = dict()604 plot_params['savepath'] = config.work_dir605 plot_params['conc_bin_centers'] = conc_bin_centers606 plot_params['r_squared'] = r_squared607 plot_params['linear_func_data'] = linear_func_data608 plot_params['well_name'] = raw_well_name609 plot_params['r'] = linear_rsquared610 plot_params['popt'] = popt611 plot_params['plot_type'] = config.plot_type612 plot_linear_rsquared_fit(plot_params)613 # now, analyze the fraction that's nucleated to determine if the system is in one or two-states614 diff = nucleated_fractions[-1] - np.min(nucleated_fractions)615 if nucleated_fractions[-1] - np.min(nucleated_fractions) < 0.15:616 mean_fret = data['damfret'].mean()617 score = (0.15-diff)/​0.15618 params.score = score619 if mean_fret < 0.05:620 params.color = 'blue'621 else:622 params.color = 'black'623 _log_class_and_score(session, well_file, params.color, params.score)624 _log_class_and_score(logger, well_file, params.color, params.score)625 return params626 # Now, fit a logistic function to the data627 popt, logistic_rs = calculate_rsquared(logistic_func, conc_bin_centers, nucleated_fractions, p0, logistic_bounds)628 saturation_conc, slope = popt629 logistic_y = logistic_func(conc_bin_centers, *popt)630 if config.plot_logistic:631 plot_params = dict()632 plot_params['savepath'] = config.work_dir633 plot_params['well_name'] = raw_well_name634 plot_params['conc_bin_centers'] = conc_bin_centers635 plot_params['nucleated_fractions'] = nucleated_fractions636 plot_params['logistic_y'] = logistic_y637 plot_params['r_squared'] = logistic_rs638 plot_params['popt'] = popt639 plot_params['plot_type'] = config.plot_type640 plot_logistic_fits(plot_params)641 # This is the final check for black.642 if saturation_conc <= config.low_conc_cutoff:643 score = (0.15-diff)/​0.15644 params.csat = saturation_conc645 params.csat_slope = slope646 params.color = 'black'647 params.score = 1.0648 _log_class_and_score(session, well_file, params.color, params.score)649 _log_class_and_score(logger, well_file, params.color, params.score)650 return params651 # Record the saturation concentration.652 logger.info('Saturation concentration: {}'.format(saturation_conc))653 if saturation_conc < max(conc_bin_centers):654 region_indices = np.where((conc_bin_centers >= saturation_conc - 1.0) & (conc_bin_centers <= saturation_conc + 1.0))655 logger.info('Saturation concentration type: lesser')656 else:657 indices = list(range(len(conc_bin_centers)))658 region_indices = indices[-5:]659 logger.info('Saturation concentration type: greater')660 region_lower = np.where(conc_bin_centers >= saturation_conc)661 region_upper = np.where(conc_bin_centers <= saturation_conc)662 _log_region_values(logger, region_indices, region_lower, region_upper, conc_bin_centers)663 # Determine if there is a large change in the fit goodness in this region. And, determine the max change in664 # fit goodness around the saturation concentration.665 region_r_squared = r_squared[region_indices]666 logger.info('Region R^2: {}'.format(region_r_squared))667 fit_value = np.max(np.abs(np.diff(region_r_squared)))668 params.csat = saturation_conc669 params.csat_slope = slope670 params.linear_r2 = linear_rsquared671 params.min_r2_region = np.min(region_r_squared)672 params.max_r2_region = np.max(region_r_squared)673 params.min_abs_diff_r2_region = np.min(np.abs(np.diff(region_r_squared)))674 params.max_abs_diff_r2_region = np.max(np.abs(np.diff(region_r_squared)))675 # check for noisy data that hasn't fully phase separated676 upper_conc_limit = config.high_conc_cutoff - 1677 if saturation_conc <= upper_conc_limit:678 df = data[data['concentration'] > saturation_conc].dropna()679 else:680 df = data[data['concentration'] > upper_conc_limit].dropna()681 df = df[df['damfret'] > 0.05].dropna()682 total_points = len(data)683 df_points = len(df)684 fraction_above_csat = df_points/​total_points685 params.frac_above_csat = fraction_above_csat686 # Finally, determine the phase separation class and its confidence score based on the687 # various parameters calculated.688 color, score = determine_class(df_points, fraction_above_csat, diff, fit_value, region_r_squared, linear_rsquared)689 params.color = color690 params.score = score691 # Log and quit.692 _log_class_and_score(session, well_file, color, score)693 _log_class_and_score(logger, well_file, color, score)694 return params695def classify_datasets(config_filename, move_to_project_root):696 """This is the main entry point for the classification of the datasets using697 a provided configuration file. This is so configured such that the user can698 use the entry point for any project directory structure.699 @param config_filename (str): The location of the YAML config file which700 will be used when analyzing the data.701 @param move_to_project_root (bool): Whether or not to move the plots from702 the `work_dir` to the `project_directory`.703 @return None704 When this function completes, the analysis (`parameters.{tsv,csv}`) is saved705 to a `work` directory saved under the `project_directory` (if no `work` directory706 is provided). A session log is also output to that location, in addition to a707 `skipped.{tsv,csv}` file which is a truncated version of `wells_filename` that708 contains all the wells which were skipped during the analysis. All plots are709 either stored in the `work` directory by default, or in the `project_directory`710 if requested by setting `move_to_project_root` to `True` when calling this function.711 """712 start_time = datetime.now()713 config = Config(config_filename)714 # Setup the logger for use.715 iso_time = start_time.strftime('%Y-%m-%d_%H-%M-%S')716 session_log_filename = os.path.join(config.work_dir, 'session---%s.log' % iso_time)717 setup_logger(config.session_name, session_log_filename, attach_stream_handler=True)718 session_log = _initialize_session_log(config)719 wells_table = pd.read_csv(config.wells_filename)720 # Calculate the classification and score for each well.721 function_args = list()722 for _index, row in wells_table.iterrows():723 raw_well_name = str(row['well_file'])724 args = (config, raw_well_name, )725 function_args.append(args)726 # Save the results /​ parameters as an `OrderedDict`727 results = parallelize(classify_dataset, function_args, config.num_processes, config.session_name)728 if move_to_project_root:729 _move_plots_to_project_root(config, session_log)730 # Reformat and save the parameters and skipped wells (if any).731 _insert_analysis_summary_section(session_log)732 parameters, skipped = _save_parameters_and_skipped_wells(config, session_log, wells_table, results)...

Full Screen

Full Screen

SessionGlobalLogger.py

Source:SessionGlobalLogger.py Github

copy

Full Screen

1import time2import copy3from settings import GlobalConfig4from session.SessionModels import (5 SessionSummary,6 SessionDeviceSummary,7 SessionTimeSummary,8 SessionApkSummary,9 SessionTestSummary,10 SessionFlakinessCheckSummary11)12from system.file import FileUtils13from system.console import (14 Printer,15 Color16)17import os18TAG = "SessionLogger:"19session_log = SessionSummary()20session_log.device_summaries = dict()21session_log.time_summary = SessionTimeSummary()22session_log.apk_summary = SessionApkSummary()23session_log.test_summary = SessionTestSummary()24session_log.test_summary.test_number = 025session_log.test_summary.test_passed = 026session_log.test_summary.test_failed = 027session_log.flakiness_summary = SessionFlakinessCheckSummary()28def _log_device(device_name):29 session_device_summary = session_log.device_summaries.get(device_name)30 if session_device_summary is None:31 session_device_summary = SessionDeviceSummary()32 session_device_summary.device_name = device_name33 session_log.device_summaries.update({device_name: session_device_summary})34def log_device_creation_start_time(device_name):35 _log_device(device_name)36 session_device_summary = session_log.device_summaries.get(device_name)37 if session_device_summary is None:38 session_device_summary = SessionDeviceSummary()39 if session_device_summary.creation_start_time is None:40 session_device_summary.creation_start_time = time.time()41 session_log.device_summaries.update({device_name: session_device_summary})42def log_device_creation_end_time(device_name):43 _log_device(device_name)44 session_device_summary = session_log.device_summaries.get(device_name)45 if session_device_summary is None:46 session_device_summary = SessionDeviceSummary()47 if session_device_summary.creation_end_time is None:48 session_device_summary.creation_end_time = time.time()49 _update_device_creation_time(session_device_summary)50 session_log.device_summaries.update({device_name: session_device_summary})51def _update_device_creation_time(session_device_summary):52 if session_device_summary.creation_start_time is not None and session_device_summary.creation_end_time is not None:53 session_device_summary.creation_time = \54 session_device_summary.creation_end_time - session_device_summary.creation_start_time55def log_device_launch_start_time(device_name):56 _log_device(device_name)57 session_device_summary = session_log.device_summaries.get(device_name)58 if session_device_summary is None:59 session_device_summary = SessionDeviceSummary()60 if session_device_summary.launch_start_time is None:61 session_device_summary.launch_start_time = time.time()62 session_log.device_summaries.update({device_name: session_device_summary})63def log_device_launch_end_time(device_name):64 _log_device(device_name)65 session_device_summary = session_log.device_summaries.get(device_name)66 if session_device_summary is None:67 session_device_summary = SessionDeviceSummary()68 if session_device_summary.launch_end_time is None:69 session_device_summary.launch_end_time = time.time()70 _update_device_launch_time(session_device_summary)71 session_log.device_summaries.update({device_name: session_device_summary})72def _update_device_launch_time(session_device_summary):73 if session_device_summary.launch_start_time is not None and session_device_summary.launch_end_time is not None:74 session_device_summary.launch_time = \75 session_device_summary.launch_end_time - session_device_summary.launch_start_time76def log_app_apk_install_start_time_on_device(device_name):77 _log_device(device_name)78 session_device_summary = session_log.device_summaries.get(device_name)79 if session_device_summary is None:80 session_device_summary = SessionDeviceSummary()81 if session_device_summary.apk_install_start_time is None:82 session_device_summary.apk_install_start_time = time.time()83 session_log.device_summaries.update({device_name: session_device_summary})84def log_app_apk_install_end_time_on_device(device_name):85 _log_device(device_name)86 session_device_summary = session_log.device_summaries.get(device_name)87 if session_device_summary is None:88 session_device_summary = SessionDeviceSummary()89 if session_device_summary.apk_install_end_time is None:90 session_device_summary.apk_install_end_time = time.time()91 _update_app_apk_install_time_on_device(session_device_summary)92 session_log.device_summaries.update({device_name: session_device_summary})93def _update_app_apk_install_time_on_device(session_device_summary):94 if session_device_summary.apk_install_start_time is not None and \95 session_device_summary.apk_install_end_time is not None:96 session_device_summary.apk_install_time = \97 session_device_summary.apk_install_end_time - session_device_summary.apk_install_start_time98def log_test_apk_install_start_time_on_device(device_name):99 _log_device(device_name)100 session_device_summary = session_log.device_summaries.get(device_name)101 if session_device_summary is None:102 session_device_summary = SessionDeviceSummary()103 if session_device_summary.test_apk_install_start_time is None:104 session_device_summary.test_apk_install_start_time = time.time()105 session_log.device_summaries.update({device_name: session_device_summary})106def log_test_apk_install_end_time_on_device(device_name):107 _log_device(device_name)108 session_device_summary = session_log.device_summaries.get(device_name)109 if session_device_summary is None:110 session_device_summary = SessionDeviceSummary()111 if session_device_summary.test_apk_install_end_time is None:112 session_device_summary.test_apk_install_end_time = time.time()113 _update_test_apk_install_time_on_device(session_device_summary)114 session_log.device_summaries.update({device_name: session_device_summary})115def _update_test_apk_install_time_on_device(session_device_summary):116 if session_device_summary.test_apk_install_end_time is not None and \117 session_device_summary.test_apk_install_start_time is not None:118 session_device_summary.test_apk_install_time = \119 session_device_summary.test_apk_install_end_time - session_device_summary.test_apk_install_start_time120def log_app_apk(app_apk_name):121 session_log.apk_summary.apk = app_apk_name122def log_test_apk(test_apk_name):123 session_log.apk_summary.test_apk = test_apk_name124def log_apk_version_code(version_code):125 session_log.apk_summary.version_code = version_code126def log_app_apk_build_start_time():127 session_log.apk_summary.apk_build_start_time = time.time()128def log_app_apk_build_end_time():129 session_log.apk_summary.apk_build_end_time = time.time()130 _update_app_apk_build_time()131def _update_app_apk_build_time():132 if session_log.apk_summary.apk_build_start_time is not None \133 and session_log.apk_summary.apk_build_end_time is not None:134 session_log.apk_summary.apk_build_time = session_log.apk_summary.apk_build_end_time - \135 session_log.apk_summary.apk_build_start_time136def log_test_apk_build_start_time():137 if session_log.apk_summary.test_apk_build_start_time is None:138 session_log.apk_summary.test_apk_build_start_time = time.time()139def log_test_apk_build_end_time():140 if session_log.apk_summary.test_apk_build_end_time is None:141 session_log.apk_summary.test_apk_build_end_time = time.time()142 _update_test_apk_build_time()143def _update_test_apk_build_time():144 if session_log.apk_summary.test_apk_build_start_time is not None \145 and session_log.apk_summary.test_apk_build_end_time is not None:146 session_log.apk_summary.test_apk_build_time = session_log.apk_summary.test_apk_build_end_time - \147 session_log.apk_summary.test_apk_build_start_time148def log_total_device_creation_start_time():149 if session_log.time_summary.total_device_creation_start_time is None:150 session_log.time_summary.total_device_creation_start_time = time.time()151def log_total_device_creation_end_time():152 if session_log.time_summary.total_device_creation_end_time is None:153 session_log.time_summary.total_device_creation_end_time = time.time()154 _update_total_device_creation_time()155def _update_total_device_creation_time():156 if session_log.time_summary.total_device_creation_start_time is not None \157 and session_log.time_summary.total_device_creation_end_time is not None:158 session_log.time_summary.total_device_creation_time = session_log.time_summary.total_device_creation_end_time \159 - \160 session_log.time_summary.total_device_creation_start_time161def log_total_device_launch_start_time():162 if session_log.time_summary.total_device_launch_start_time is None:163 session_log.time_summary.total_device_launch_start_time = time.time()164def log_total_device_launch_end_time():165 if session_log.time_summary.total_device_launch_end_time is None:166 session_log.time_summary.total_device_launch_end_time = time.time()167 _update_total_device_launch_time()168def _update_total_device_launch_time():169 if session_log.time_summary.total_device_launch_start_time is not None \170 and session_log.time_summary.total_device_launch_end_time is not None:171 session_log.time_summary.total_device_launch_time = session_log.time_summary.total_device_launch_end_time - \172 session_log.time_summary.total_device_launch_start_time173def log_total_apk_build_start_time():174 if session_log.time_summary.total_apk_build_start_time is None:175 session_log.time_summary.total_apk_build_start_time = time.time()176def log_total_apk_build_end_time():177 if session_log.time_summary.total_apk_build_end_time is None:178 session_log.time_summary.total_apk_build_end_time = time.time()179 _update_total_apk_build_time()180def _update_total_apk_build_time():181 if session_log.time_summary.total_apk_build_start_time is not None \182 and session_log.time_summary.total_apk_build_end_time is not None:183 session_log.time_summary.total_apk_build_time = \184 session_log.time_summary.total_apk_build_end_time - session_log.time_summary.total_apk_build_start_time185def log_total_apk_install_start_time():186 if session_log.time_summary.total_apk_install_start_time is None:187 session_log.time_summary.total_apk_install_start_time = time.time()188def log_total_apk_install_end_time():189 if session_log.time_summary.total_apk_install_end_time is None:190 session_log.time_summary.total_apk_install_end_time = time.time()191 _update_total_apk_install_time()192def _update_total_apk_install_time():193 if session_log.time_summary.total_apk_install_start_time is not None \194 and session_log.time_summary.total_apk_install_end_time is not None:195 session_log.time_summary.total_apk_install_time = \196 session_log.time_summary.total_apk_install_end_time - session_log.time_summary.total_apk_install_start_time197def log_total_test_start_time():198 if session_log.time_summary.total_test_start_time is None:199 session_log.time_summary.total_test_start_time = time.time()200def log_total_test_end_time():201 if session_log.time_summary.total_test_end_time is None:202 session_log.time_summary.total_test_end_time = time.time()203 _update_total_test_time()204def _update_total_test_time():205 if session_log.time_summary.total_test_start_time is not None \206 and session_log.time_summary.total_test_end_time is not None:207 session_log.time_summary.total_test_time = \208 session_log.time_summary.total_test_end_time - session_log.time_summary.total_test_start_time209def log_total_rerun_start_time():210 if session_log.time_summary.total_rerun_start_time is None:211 session_log.time_summary.total_rerun_start_time = time.time()212def log_total_rerun_end_time():213 if session_log.time_summary.total_rerun_end_time is None:214 session_log.time_summary.total_rerun_end_time = time.time()215 _update_total_rerun_time()216def _update_total_rerun_time():217 if session_log.time_summary.total_rerun_start_time is not None \218 and session_log.time_summary.total_rerun_end_time is not None:219 session_log.time_summary.total_rerun_time = \220 session_log.time_summary.total_rerun_end_time - session_log.time_summary.total_rerun_start_time221def log_session_start_time():222 if session_log.time_summary.total_session_start_time is None:223 session_log.time_summary.total_session_start_time = time.time()224def log_session_end_time():225 if session_log.time_summary.total_session_end_time is None:226 session_log.time_summary.total_session_end_time = time.time()227 _update_total_session_time()228def _update_total_session_time():229 if session_log.time_summary.total_session_start_time is not None \230 and session_log.time_summary.total_session_end_time is not None:231 session_log.time_summary.total_session_time = \232 session_log.time_summary.total_session_end_time - session_log.time_summary.total_session_start_time233def increment_passed_tests():234 session_log.test_summary.test_passed += 1235 _update_test_number()236 _update_health_rate()237def increment_failed_tests():238 session_log.test_summary.test_failed += 1239 _update_test_number()240 _update_health_rate()241def _update_test_number():242 session_log.test_summary.test_number = session_log.test_summary.test_passed + session_log.test_summary.test_failed243def _update_health_rate():244 if session_log.test_summary.test_passed != 0 and session_log.test_summary.test_failed == 0:245 session_log.test_summary.health_rate = 1.00246 elif session_log.test_summary.test_passed == 0 and session_log.test_summary.test_failed != 0:247 session_log.test_summary.health_rate = 0.00248 elif session_log.test_summary.test_passed == 0 and session_log.test_summary.test_failed == 0:249 session_log.test_summary.health_rate = 1.00250 else:251 session_log.test_summary.health_rate = session_log.test_summary.test_passed /​ \252 session_log.test_summary.test_number253def update_flaky_candidate(test_summary):254 session_log.flakiness_summary.suspects.setdefault(test_summary.test_name, {255 "failed_count": 1,256 "passed_count": 0,257 "is_flaky": False258 })259 if test_summary.test_status == "success":260 session_log.flakiness_summary.suspects[test_summary.test_name]["passed_count"] += 1261 session_log.flakiness_summary.suspects[test_summary.test_name]["is_flaky"] = True262 else:263 session_log.flakiness_summary.suspects[test_summary.test_name]["failed_count"] += 1264def dump_session_summary():265 print_device_summaries()266 print_time_summary()267 print_apk_summary()268 print_test_summary()269 print_rerun_summary()270def print_device_summaries():271 if session_log.device_summaries:272 Printer.system_message(TAG, "Device details:")273 for key, device_summary in session_log.device_summaries.items():274 Printer.system_message(TAG, " - device " + Color.GREEN + device_summary.device_name + Color.BLUE + ":")275 if device_summary.creation_time is not None:276 Printer.system_message(TAG, " * Creation time: " + Color.GREEN + "{:.2f}".format(277 device_summary.creation_time) + Color.BLUE + " seconds.")278 if device_summary.launch_time is not None:279 Printer.system_message(TAG, " * Launch time: " + Color.GREEN + "{:.2f}".format(280 device_summary.launch_time) + Color.BLUE + " seconds.")281 if device_summary.apk_install_time is not None:282 Printer.system_message(TAG, " * .*apk install time: " + Color.GREEN + "{:.2f}".format(283 device_summary.apk_install_time) + Color.BLUE + " seconds.")284 if device_summary.test_apk_install_time is not None:285 Printer.system_message(TAG, " * Test .*apk install time: " + Color.GREEN + "{:.2f}".format(286 device_summary.test_apk_install_time) + Color.BLUE + " seconds.")287def print_time_summary():288 Printer.system_message(TAG, "Time details:")289 if session_log.time_summary.total_device_creation_time is not None:290 Printer.system_message(TAG, " * Total device creation process took: " + Color.GREEN +291 "{:.2f}".format(session_log.time_summary.total_device_creation_time) + Color.BLUE292 + " seconds.")293 if session_log.time_summary.total_device_launch_time is not None:294 Printer.system_message(TAG, " * Total device launch process took: " + Color.GREEN +295 "{:.2f}".format(session_log.time_summary.total_device_launch_time) + Color.BLUE296 + " seconds.")297 if session_log.time_summary.total_apk_build_time is not None:298 Printer.system_message(TAG, " * Total .*apk build process took: " + Color.GREEN +299 "{:.2f}".format(session_log.time_summary.total_apk_build_time) + Color.BLUE300 + " seconds.")301 if session_log.time_summary.total_apk_install_time is not None:302 Printer.system_message(TAG, " * Total .*apk installation process took: " + Color.GREEN +303 "{:.2f}".format(session_log.time_summary.total_apk_install_time) + Color.BLUE304 + " seconds.")305 if session_log.time_summary.total_test_time is not None:306 Printer.system_message(TAG, " * Total test process took: " + Color.GREEN +307 "{:.2f}".format(session_log.time_summary.total_test_time) + Color.BLUE + " seconds.")308 if session_log.time_summary.total_rerun_time is not None:309 Printer.system_message(TAG, " * Total test re-run process took: " + Color.GREEN +310 "{:.2f}".format(session_log.time_summary.total_rerun_time) + Color.BLUE + " seconds.")311 if session_log.time_summary.total_session_time is not None:312 Printer.system_message(TAG, " * Total session time: " + Color.GREEN +313 "{:.2f}".format(session_log.time_summary.total_session_time) + Color.BLUE + " seconds.")314def print_apk_summary():315 if session_log.apk_summary.apk is not None and session_log.apk_summary.apk_build_time is not None:316 Printer.system_message(TAG, ".*apk details:")317 Printer.system_message(TAG, " * Application .*apk name: " + Color.GREEN + session_log.apk_summary.apk318 + Color.BLUE + ".")319 if session_log.apk_summary.apk_build_time is not None:320 Printer.system_message(TAG, " * Application .*apk build and scan time: " + Color.GREEN321 + "{:.2f}".format(session_log.apk_summary.apk_build_time) + Color.BLUE + " seconds.")322 if session_log.apk_summary.test_apk is not None:323 Printer.system_message(TAG, " * Test .*apk name: " + Color.GREEN + session_log.apk_summary.test_apk324 + Color.BLUE + ".")325 if session_log.apk_summary.test_apk_build_time is not None:326 Printer.system_message(TAG, " * Test .*apk build and scan time: " + Color.GREEN327 + "{:.2f}".format(session_log.apk_summary.test_apk_build_time) + Color.BLUE328 + " seconds.")329 if session_log.apk_summary.version_code is not None:330 Printer.system_message(TAG, " * Version: " + Color.GREEN + str(session_log.apk_summary.version_code)331 + Color.BLUE + ".")332def print_test_summary():333 Printer.system_message(TAG, "Test details:")334 Printer.system_message(TAG, " * Total number of test cases: " + Color.GREEN +335 str(session_log.test_summary.test_number) + Color.BLUE + ".")336 Printer.system_message(TAG, " * Tests passed: " + Color.GREEN + str(session_log.test_summary.test_passed)337 + Color.BLUE + ".")338 Printer.system_message(TAG, " * Tests failed: " + Color.GREEN + str(session_log.test_summary.test_failed)339 + Color.BLUE + ".")340 if session_log.test_summary.health_rate is not None:341 Printer.system_message(TAG, " * Health rate: " + Color.GREEN342 + "{0:.2f}%".format(session_log.test_summary.health_rate * 100) + Color.BLUE + ".")343def print_rerun_summary():344 if GlobalConfig.SHOULD_RERUN_FAILED_TESTS:345 Printer.system_message(TAG, "Re-run details:")346 Printer.system_message(TAG, " * from " + Color.GREEN + "{}".format(session_log.test_summary.test_failed)347 + Color.BLUE + " failed test cases, each was started again " + Color.GREEN348 + "{}".format(GlobalConfig.FLAKINESS_RERUN_COUNT) + Color.BLUE + " times:")349 for suspect, status in session_log.flakiness_summary.suspects.items():350 Printer.system_message(TAG, " * (failed: " + Color.GREEN +351 "{}".format(status["failed_count"]) + Color.BLUE + ", passed: " + Color.GREEN352 + "{}".format(status["passed_count"]) + Color.BLUE + ", is_flaky: " + Color.GREEN353 + "{}".format(status["is_flaky"]) + Color.BLUE + ") {}".format(suspect))354def dump_saved_files_history():355 nothing_to_display = True356 if FileUtils.dir_exists(GlobalConfig.OUTPUT_AVD_LOG_DIR):357 nothing_to_display = False358 saved_avd_logs = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_AVD_LOG_DIR)359 Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_AVD_LOG_DIR)360 + Color.BLUE + " (" + Color.GREEN + str(len(saved_avd_logs)) + " files" + Color.BLUE361 + ")")362 for saved_file in FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_AVD_LOG_DIR):363 Printer.system_message(TAG, " * " + saved_file + ".")364 if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_LOG_DIR):365 nothing_to_display = False366 saved_test_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOG_DIR)367 Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_LOG_DIR)368 + Color.BLUE + " (" + Color.GREEN + str(len(saved_test_summaries)) + " files"369 + Color.BLUE + ")")370 for saved_file in FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOG_DIR):371 Printer.system_message(TAG, " * " + saved_file + ".")372 if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR):373 nothing_to_display = False374 saved_logcats_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR)375 Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_LOGCAT_DIR)376 + Color.BLUE + " (" + Color.GREEN + str(len(saved_logcats_summaries)) + " files"377 + Color.BLUE + ")")378 for saved_file in saved_logcats_summaries:379 Printer.system_message(TAG, " * " + saved_file + ".")380 if GlobalConfig.SHOULD_RECORD_TESTS:381 if FileUtils.dir_exists(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR):382 nothing_to_display = False383 saved_recordings_summaries = FileUtils.list_files_in_dir(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR)384 Printer.system_message(TAG, "Directory " + Color.GREEN + str(GlobalConfig.OUTPUT_TEST_RECORDINGS_DIR)385 + Color.BLUE + " (" + Color.GREEN + str(len(saved_recordings_summaries))386 + " files" + Color.BLUE + ")")387 for saved_file in saved_recordings_summaries:388 Printer.system_message(TAG, " * " + saved_file + ".")389 if nothing_to_display:390 Printer.system_message(TAG, "No files were saved during session.")391def save_session_summary():392 if GlobalConfig.OUTPUT_SUMMARY_LOG_DIR is not None and os.path.exists(GlobalConfig.OUTPUT_SUMMARY_LOG_DIR):393 session_log_json_dict = copy.deepcopy(session_log)394 session_log_json_dict.time_summary = vars(session_log.time_summary)395 session_log_json_dict.apk_summary = vars(session_log.apk_summary)396 session_log_json_dict.test_summary = vars(session_log.test_summary)397 session_log_json_dict.device_summaries = list()398 for device_name, device_summary in session_log.device_summaries.items():399 session_log_json_dict.device_summaries.append(vars(device_summary))400 session_log_json_dict.flakiness_summary = vars(session_log.flakiness_summary)401 session_log_json_dict = vars(session_log_json_dict)...

Full Screen

Full Screen

irc_extractor.py

Source:irc_extractor.py Github

copy

Full Screen

1import json2import os3import socket4from collections import defaultdict5from copy import deepcopy6import dpkt7import numpy as np8import re9import irc_config10def compute_session_periodicity(communication):11 """12 using Fast Fourier Transform to compute periodicity of messages in sessions13 @:return value in [0,1] interval - 0 means that messages are not periodic at all14 """15 if len(communication) < 3:16 return None17 t = list(map(lambda x: x['timestamp'], communication))18 td = np.asarray(list(map(lambda x: x[0] - x[1], zip(t[1:], t))))19 fft_res = np.absolute(np.fft.fft(td))20 T = fft_res.argmax() + 221 rng_size = int(len(td) /​ T)22 td_T = [td[x * T:x * T + T] for x in range(rng_size)]23 td_T_avg = np.mean(td_T, 0)24 # ||td_t - td_avg ||2 /​ ||td_t||25 td_nmse = np.linalg.norm(td_T - td_T_avg) /​ np.linalg.norm(td_T)26 return 1 - td_nmse27def compute_msg_periodicity(communication):28 if len(communication) < 3:29 return len(communication) * [None]30 t = list(map(lambda x: x['timestamp'], communication))31 msg_per = list()32 # not able to compute last element periodicity - no successor msg33 msg_per.append(None)34 for ta, tb, tc in zip(t, t[1:], t[2:]):35 t1 = tb - ta36 t2 = tc - tb37 try:38 msg_per.append(t2 /​ t1)39 except ZeroDivisionError:40 msg_per.append(None)41 print('t1 = {} - {} = {}, t2:{} - {} = {},'.format(tb, ta, t1, tc, tb, t2))42 # not able to compute last element periodicity - no successor msg43 msg_per.append(None)44 return msg_per45class IRCExtractor:46 irc_port_dict = {'2.irc': 2407, '3.irc': 2407, '4.irc': 6667, '34.irc': 2407, '39.irc': 6667, '42.irc': 4975,47 '51.irc': 54468, '56.irc': 80, '62.irc': 443, 'irc1': 6667, 'irc3': 6667}48 def __init__(self, pcap):49 self.irc_logs = defaultdict(lambda: [])50 self.irc_packet_counter = 051 self.unfinished_msg = ''52 self.msg_not_finished = False53 self.irc_port = self.irc_port_dict[pcap]54 def process_packet(self, timestamp, buffer):55 eth = dpkt.ethernet.Ethernet(buffer)56 if eth.type != dpkt.ethernet.ETH_TYPE_IP and eth.type != dpkt.ethernet.ETH_TYPE_8021Q:57 # not ip packet58 return59 try:60 ip = eth.data61 # not tcp packet62 if type(ip.data) != dpkt.tcp.TCP:63 return64 except AttributeError:65 return 66 67 try:68 tcp = ip.data69 ip_src = socket.inet_ntoa(ip.src)70 ip_dst = socket.inet_ntoa(ip.dst)71 sport = tcp.sport72 dport = tcp.dport73 data = tcp.data74 except OSError:75 return76 if sport != self.irc_port and dport != self.irc_port:77 # invalid port78 return79 if self.irc_packet_counter % 1000 == 0:80 print('IRC PRIVMSG Packet #{}'.format(self.irc_packet_counter))81 try:82 payload = data.decode('ascii')83 except:84 return85 self.irc_packet_counter += 186 if data and 'PRIVMSG' in payload:87 # payload = data.decode('utf8')88 # payload schema is (list of) src PRIVMSG dst msg \r\n, so we split them by \r\n89 p_splitted = payload.split('\r\n')90 p_len = len(p_splitted)91 for i, msg in enumerate(p_splitted):92 if i == 0 and self.msg_not_finished:93 self.msg_not_finished = False94 msg = self.unfinished_msg + msg95 if i == p_len - 1 and payload[:4] != '\r\n':96 self.msg_not_finished = True97 self.unfinished_msg = msg98 return99 msg_splitted = msg.split()100 if len(msg_splitted) < 4 or msg_splitted[1] != 'PRIVMSG':101 continue102 # msg starts with :, so it can be neglected103 src, dst, msg_text = msg_splitted[0], msg_splitted[2], ' '.join(msg_splitted[3:])[1:]104 irc_log = {'timestamp': timestamp, 'msg': msg_text, 'pkt_size': len(buffer), 'sport': sport}105 self.irc_logs[((src, ip_src), (dst, ip_dst, dport))].append(irc_log)106 def save_logs(self, pcap_out_json_path, malicious):107 logs = []108 for session, communication in self.irc_logs.items():109 session_log = {}110 src, dst = session[0], session[1]111 msg_times = list(map(lambda c: c['timestamp'], communication))112 pkt_sizes = list(map(lambda c: c['pkt_size'], communication))113 session_log['src'] = src[0]114 session_log['src_ip'] = src[1]115 session_log['src_ports'] = list(set(map(lambda x: x['sport'], communication)))116 for msg in communication:117 del msg['sport']118 119 session_log['dst'] = dst[0]120 session_log['dst_ip'] = dst[1]121 session_log['dst_port'] = dst[2]122 session_log['start_time'] = min(msg_times)123 session_log['end_time'] = max(msg_times)124 session_log['duration'] = session_log['end_time'] - session_log['start_time']125 session_log['msg_count'] = len(communication)126 session_log['pkt_size_total'] = sum(pkt_sizes)127 per = compute_session_periodicity(communication)128 if per is not None:129 session_log['periodicity'] = per130 else:131 session_log['periodicity'] = np.nan132 msg_per = compute_msg_periodicity(communication)133 134 # MODEL 2135 src = src[0]136 usr_rgx = re.match(r'^.*?(?=@)', src + '@')137 src_username = src[: usr_rgx.regs[0][1]]138 spec_chars_username_mean = 0 if len(src_username) == 0 else len(re.findall(r'[^A-Za-z]', src_username)) /​ len(src_username)139 msg_special_chars = []140 msg_word_count = defaultdict(lambda: 0)141 comm2 = []142 for msg, p in zip(communication, msg_per):143 msg['periodicity'] = p if p is not None else np.nan144 msg_content = msg['msg']145 msg_words = msg_content.split()146 for word in msg_words:147 msg_word_count[word] += 1148 msg_spec = 0 if len(msg_content) == 0 else len(re.findall(r'[^A-Za-z]', msg_content)) /​ len(msg_content)149 msg_special_chars.append(msg_spec)150 151 comm2.append(msg)152 _wordcounts = list(msg_word_count.values())153 p = _wordcounts /​ np.sum(_wordcounts)154 msg_word_entropy = -np.sum(p * np.log2(p))155 spec_chars_msg_mean = np.mean(msg_special_chars)156 session_log['spec_chars_username_mean'] = spec_chars_username_mean157 session_log['spec_chars_msg_mean'] = spec_chars_msg_mean158 session_log['msg_word_entropy'] = msg_word_entropy159 session_log['malicious'] = malicious160 session_log['msgs'] = comm2161 logs.append(session_log)162 final_irc_logs = {'sessions': logs}163 self.irc_logs = logs164 with open(pcap_out_json_path, 'w+', encoding='utf-8') as f:165 f.write(json.dumps(final_irc_logs, indent=4))166 def filter_logs(self, irc_logs, pcap_json_path):167 logs = []168 for session, communication in irc_logs.items():169 filtered_com = []170 for c in communication:171 if c['src'] != 'PRIVMSG' and c['src'] != 'NOTICE' and c['dst'] != 'PRIVMSG' and c['dst'] != 'NOTICE':172 filtered_com.append(c)173 if len(filtered_com) > 0:174 logs.append({session: filtered_com})175 final_irc_logs = {'data': logs}176 with open(pcap_json_path, 'w+', encoding='utf-8') as f:177 f.write(json.dumps(final_irc_logs, indent=4))178 def load_logs(self, json_filename):179 print('loading logs...')180 self.irc_logs = defaultdict(lambda: [])181 with open(json_filename, 'r') as f:182 self.irc_logs = json.load(f)['sessions']183 return deepcopy(self.irc_logs)184 def sniff_pcap(self, pcap_path, pcap_out_json_path, malicious):185 print('sniffing pcap...')186 self.irc_logs = defaultdict(lambda: [])187 with open(pcap_path, 'rb') as f:188 pcap = dpkt.pcap.Reader(f)189 for ts, buf in pcap:190 # checking for zero ethertype and throwing away:191 if len(buf) > 0:192 try:193 self.process_packet(ts, buf)194 except:195 continue196 197 self.save_logs(pcap_out_json_path, malicious)198 return self.irc_logs199def main():200 for pcap in irc_config.PCAPS.items():201 print('Sniffing PCAP {}...'.format(pcap[0]))202 irc_extractor = IRCExtractor(pcap[0])203 if os.path.isfile(irc_config.pcap_json_path(pcap[0])):204 print(irc_config.pcap_json_path(pcap[0]))205 irc_logs = irc_extractor.load_logs(irc_config.pcap_json_path(pcap[0]))206 # filter_logs()207 else:208 irc_logs = irc_extractor.sniff_pcap(irc_config.pcap_path(pcap), irc_config.pcap_json_path(pcap[0]), malicious=pcap[1])209 # irc_extractor.extract_features()210 # irc_logs.sort(key=lambda x: x.get('timestamp'))211 # graph = build_graph(irc_logs)212 # visualize_graph(graph, PCAP_GRAPH_PATH)213if __name__ == '__main__':...

Full Screen

Full Screen

test_netmiko_sesssion_log.py

Source:test_netmiko_sesssion_log.py Github

copy

Full Screen

...14 contents = f.read()15 else:16 raise ValueError("Most specify either file_name or contents")17 return hashlib.md5(contents).hexdigest()18def read_session_log(session_file, append=False):19 """Leading white-space can vary. Strip off leading white-space."""20 with open(session_file, "rb") as f:21 if append is True:22 line = f.readline().decode()23 assert "Initial file contents" in line24 log_content = f.read().lstrip()25 return log_content26def session_action(my_connect, command):27 """Common actions in the netmiko session to generate the session log."""28 time.sleep(1)29 my_connect.clear_buffer()30 output = my_connect.send_command(command)31 my_connect.disconnect()32 return output33def session_log_md5(session_file, compare_file):34 """Compare the session_log MD5 to the compare_file MD5"""35 compare_log_md5 = calc_md5(file_name=compare_file)36 log_content = read_session_log(session_file)37 session_log_md5 = calc_md5(contents=log_content)38 assert session_log_md5 == compare_log_md539def session_log_md5_append(session_file, compare_file):40 """Compare the session_log MD5 to the compare_file MD5"""41 compare_log_md5 = calc_md5(file_name=compare_file)42 log_content = read_session_log(session_file, append=True)43 session_log_md5 = calc_md5(contents=log_content)44 assert session_log_md5 == compare_log_md545def test_session_log(net_connect, commands, expected_responses):46 """Verify session_log matches expected content."""47 command = commands["basic"]48 session_action(net_connect, command)49 compare_file = expected_responses["compare_log"]50 session_file = expected_responses["session_log"]51 session_log_md5(session_file, compare_file)52def test_session_log_write(net_connect_slog_wr, commands, expected_responses):53 """Verify session_log matches expected content, but when channel writes are also logged."""54 command = commands["basic"]55 session_action(net_connect_slog_wr, command)56 compare_file = expected_responses["compare_log_wr"]57 session_file = expected_responses["session_log"]58 session_log_md5(session_file, compare_file)59def test_session_log_append(device_slog, commands, expected_responses):...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Slash automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

... View sample repo... View DocumentationSign up Free
›
_

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

...
Sign up with GoogleSign up with Email
Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful