How to use _load_and_merge method in prospector

Best Python code snippet using prospector_python

benchmark_result.py

Source:benchmark_result.py

...140        else:141            configs_idx_to_load = None # all available histories are loaded142        new_histories = None143        with zipfile.ZipFile(self.__file_name, 'r') as result_zip_file:144            new_histories, configs_not_found, processed_files = BenchmarkResult._load_and_merge(result_zip_file, \145                "histories", configs_idx_to_load)146        if len(configs_not_found) > 0:147            logging.warning("The histories with config indices {} were not found in {}".format(configs_not_found, self.__file_name))148        self.__histories.update(new_histories)149        return processed_files150    def load_benchmark_configs(self, config_idx_list = None):151        if config_idx_list:152            existing_configs_indices = [bc.config_idx for bc in self.__benchmark_configs]153            configs_idx_to_load = list(set(config_idx_list) - set(existing_configs_indices))154        else:155            configs_idx_to_load = None # all available configs are loaded156        new_bench_configs = None157        with zipfile.ZipFile(self.__file_name, 'r') as result_zip_file:158            new_bench_configs, configs_not_found, processed_files = BenchmarkResult._load_and_merge(result_zip_file, \159                "configs", configs_idx_to_load)160            new_bench_configs_list = list(new_bench_configs.values())161        if len(configs_not_found) > 0:162            logging.warning("The benchmark configs with indices {} were not found in {}".format(configs_not_found, self.__file_name))163        self.__benchmark_configs.extend(new_bench_configs_list)164        return processed_files165    @staticmethod166    def _find_files_to_load(total_file_list, filetype, config_idx_list):167        files_to_load = []168        configs_not_found = set(config_idx_list)169        for file in total_file_list:170            match = re.search("\d+(?:,\d+)*".format(filetype), file)171            if not match:172                continue173            matched_index_list = match.group(0)174            file_configs = [int(idx) for idx in matched_index_list.split(",")]175            found_config = list(filter(lambda conf_idx: conf_idx in file_configs, config_idx_list))176            configs_not_found -= set(found_config)177            if len(found_config) > 0:178                files_to_load.append(file)179        return files_to_load, configs_not_found180    @staticmethod181    def _load_and_merge(zip_file_handle, filetype, config_idx_list):182        total_file_list = [filename for filename in zip_file_handle.namelist() \183                        if filetype in filename]184        configs_not_found = []185        if len(total_file_list) < 1:  186            logging.warning("There are no files for type: {}. Have you forgotten to specify it in dump()?".format(filetype))187        files_to_load = total_file_list188        if config_idx_list:189            files_to_load, configs_not_found = BenchmarkResult._find_files_to_load(total_file_list, \190                filetype, config_idx_list)191        merged_iterable_dict = {}192        for file in files_to_load:193            bytes = zip_file_handle.read(file)194            iterable = pickle.loads(bytes)195            merged_iterable_dict.update(iterable)...

profile.py

Source:profile.py

...88        return yaml.safe_dump(self.as_dict())89    @staticmethod90    def load(name_or_path, profile_path, allow_shorthand=True, forced_inherits=None):91        # First simply load all of the profiles and those that it explicitly inherits from92        data, inherits = _load_and_merge(93            name_or_path,94            profile_path,95            allow_shorthand,96            forced_inherits=forced_inherits or [],97        )98        return ProspectorProfile(name_or_path, data, inherits)99def _is_valid_extension(filename):100    ext = os.path.splitext(filename)[1]101    return ext in (".yml", ".yaml")102def _load_content(name_or_path, profile_path):103    filename = None104    if _is_valid_extension(name_or_path):105        for path in profile_path:106            filepath = os.path.join(path, name_or_path)107            if os.path.exists(filepath):108                # this is a full path that we can load109                filename = filepath110                break111        if filename is None:112            raise ProfileNotFound(name_or_path, profile_path)113    else:114        for path in profile_path:115            for ext in ("yml", "yaml"):116                filepath = os.path.join(path, "%s.%s" % (name_or_path, ext))117                if os.path.exists(filepath):118                    filename = filepath119                    break120        if filename is None:121            raise ProfileNotFound(name_or_path, profile_path)122    with open(filename) as fct:123        try:124            return yaml.safe_load(fct) or {}125        except yaml.parser.ParserError as parse_error:126            raise CannotParseProfile(filename, parse_error)127def _ensure_list(value):128    if isinstance(value, list):129        return value130    return [value]131def _simple_merge_dict(priority, base):132    out = dict(base.items())133    out.update(dict(priority.items()))134    return out135def _merge_tool_config(priority, base):136    out = dict(base.items())137    for key, value in priority.items():138        # pep8 has extra 'full' and 'none' options139        # pylint has extra 'load-plugins' option140        if key in ("run", "full", "none", "load-plugins"):141            out[key] = value142        elif key in ("options",):143            out[key] = _simple_merge_dict(value, base.get(key, {}))144    # anything enabled in the 'priority' dict is removed145    # from 'disabled' in the base dict and vice versa146    base_disabled = base.get("disable") or []147    base_enabled = base.get("enable") or []148    pri_disabled = priority.get("disable") or []149    pri_enabled = priority.get("enable") or []150    out["disable"] = list(set(pri_disabled) | (set(base_disabled) - set(pri_enabled)))151    out["enable"] = list(set(pri_enabled) | (set(base_enabled) - set(pri_disabled)))152    return out153def _merge_profile_dict(priority, base):154    # copy the base dict into our output155    out = dict(base.items())156    for key, value in priority.items():157        if key in (158            "strictness",159            "doc-warnings",160            "test-warnings",161            "member-warnings",162            "output-format",163            "autodetect",164            "max-line-length",165        ):166            # some keys are simple values which are overwritten167            out[key] = value168        elif key in (169            "ignore",170            "ignore-patterns",171            "ignore-paths",172            "uses",173            "requirements",174            "python-targets",175            "output-target",176        ):177            # some keys should be appended178            out[key] = _ensure_list(value) + _ensure_list(base.get(key, []))179        elif key in TOOLS.keys():180            # this is tool config!181            out[key] = _merge_tool_config(value, base.get(key, {}))182    return out183def _determine_strictness(profile_dict, inherits):184    for profile in inherits:185        if profile.startswith("strictness_"):186            return None, False187    strictness = profile_dict.get("strictness")188    if strictness is None:189        return None, False190    return ("strictness_%s" % strictness), True191def _determine_pep8(profile_dict):192    pep8 = profile_dict.get("pep8", {})193    if pep8.get("full", False):194        return "full_pep8", True195    elif pep8.get("none", False):196        return "no_pep8", True197    return None, False198def _determine_doc_warnings(profile_dict):199    doc_warnings = profile_dict.get("doc-warnings")200    if doc_warnings is None:201        return None, False202    return ("doc_warnings" if doc_warnings else "no_doc_warnings"), True203def _determine_test_warnings(profile_dict):204    test_warnings = profile_dict.get("test-warnings")205    if test_warnings is None:206        return None, False207    return (None if test_warnings else "no_test_warnings"), True208def _determine_member_warnings(profile_dict):209    member_warnings = profile_dict.get("member-warnings")210    if member_warnings is None:211        return None, False212    return ("member_warnings" if member_warnings else "no_member_warnings"), True213def _determine_implicit_inherits(profile_dict, already_inherits, shorthands_found):214    # Note: the ordering is very important here - the earlier items215    # in the list have precedence over the later items. The point of216    # the doc/test/pep8 profiles is usually to restore items which were217    # turned off in the strictness profile, so they must appear first.218    implicit = [219        ("pep8", _determine_pep8(profile_dict)),220        ("docs", _determine_doc_warnings(profile_dict)),221        ("tests", _determine_test_warnings(profile_dict)),222        ("strictness", _determine_strictness(profile_dict, already_inherits)),223        ("members", _determine_member_warnings(profile_dict)),224    ]225    inherits = []226    for shorthand_name, determined in implicit:227        if shorthand_name in shorthands_found:228            continue229        extra_inherits, shorthand_found = determined230        if not shorthand_found:231            continue232        shorthands_found.add(shorthand_name)233        if extra_inherits is not None:234            inherits.append(extra_inherits)235    return inherits, shorthands_found236def _append_profiles(name, profile_path, data, inherit_list, allow_shorthand=False):237    new_data, new_il, _ = _load_profile(name, profile_path, allow_shorthand=allow_shorthand)238    data.update(new_data)239    inherit_list += new_il240    return data, inherit_list241def _load_and_merge(name_or_path, profile_path, allow_shorthand=True, forced_inherits=None):242    # First simply load all of the profiles and those that it explicitly inherits from243    data, inherit_list, shorthands_found = _load_profile(244        name_or_path,245        profile_path,246        allow_shorthand=allow_shorthand,247        forced_inherits=forced_inherits or [],248    )249    if allow_shorthand:250        if "docs" not in shorthands_found:251            data, inherit_list = _append_profiles("no_doc_warnings", profile_path, data, inherit_list)252        if "members" not in shorthands_found:253            data, inherit_list = _append_profiles("no_member_warnings", profile_path, data, inherit_list)254        if "tests" not in shorthands_found:255            data, inherit_list = _append_profiles("no_test_warnings", profile_path, data, inherit_list)...

inputparser.py

Source:inputparser.py

...20    """21    def __init__(self, patterns: List[str]):22        self.patterns = patterns23        self.filenames = self._resolve_patterns(patterns)24        self.data = self._load_and_merge(self.filenames)25    def _load_and_merge(self, filenames: List[str]):26        """27        Load files and merge them into a big dictionary28        """29        if not filenames:30            print("No input files available!")31            return {}32        merged_data = {}33        for filename in filenames:34            with open(filename, "r") as fp:35                try:36                    data = load(fp.read(), Loader=Loader)37                except Exception as e:38                    print(f"Failed to open file {filename} with error: {e}")39                    continue...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.