How to use absolute_paths method in prospector

Best Python code snippet using prospector_python

paths.py

Source:paths.py

1import importlib2from pathlib import Path3from typing import Callable, List, NewType, Type4from fs.base import FS5from fs.osfs import OSFS6AbsolutePath = NewType("AbsolutePath", Path)7def load_class_by_full_path(path_to_mapper_obj: str) -> Type:8    *path, name = path_to_mapper_obj.split(".")9    try:10        module = importlib.import_module(".".join(path))11    except ModuleNotFoundError as e:12        raise ValueError(f'Module {".".join(path)} not defined.') from e13    except ValueError as e:14        raise ValueError(f"Invalid full path: {path_to_mapper_obj}") from e15    try:16        return getattr(module, name)17    except AttributeError as e:18        raise ValueError(f"Class {name} not found in module {module}") from e19def normalize_paths(20    paths: List[str], fs: FS, predicate: Callable[[Path], bool]21) -> List[str]:22    """23    >>> from fs.memoryfs import MemoryFS24    >>> fs = MemoryFS()25    >>> _ = fs.makedirs('root')26    >>> _ = fs.makedirs('root/dir1')27    >>> _ = fs.makedirs('root/dir2')28    >>> _ = fs.touch('root/file0.txt')29    >>> fs.touch('root/dir1/file11.txt')30    >>> fs.touch('root/dir1/file12.txt')31    >>> fs.touch('root/dir2/file21.txt')32    >>> fs.touch('root/dir2/file22.txt')33    >>> absolute_paths = normalize_paths(['root/file0.txt'], fs, lambda x: True)34    >>> [str(Path(path)) for path in absolute_paths]35    ['root/file0.txt']36    >>> absolute_paths = normalize_paths(['root/file0.txt', 'root/dir1/file11.txt'], fs, lambda x: True)37    >>> [str(Path(path)) for path in absolute_paths]38    ['root/dir1/file11.txt', 'root/file0.txt']39    >>> absolute_paths = normalize_paths(['root/file0.txt', 'root/dir1/file11.txt', 'root/dir1/file12.txt'], fs, lambda x: True)40    >>> [str(Path(path)) for path in absolute_paths]41    ['root/dir1', 'root/file0.txt']42    >>> absolute_paths = normalize_paths(['root/file0.txt', 'root/dir1', 'root/dir1/file12.txt'], fs, lambda x: True)43    >>> [str(Path(path)) for path in absolute_paths]44    ['root/dir1', 'root/file0.txt']45    >>> absolute_paths = normalize_paths(['root/file0.txt', 'root/dir1', 'root/dir1/file11.txt', 'root/dir1/file12.txt'], fs, lambda x: True)46    >>> [str(Path(path)) for path in absolute_paths]47    ['root/dir1', 'root/file0.txt']48    """49    non_collapsable = set()50    absolute_paths = [Path(path) for path in paths]51    grouped = {}52    for path in absolute_paths:53        if path.parent not in grouped:54            grouped[path.parent] = set()55        grouped[path.parent].add(path.name)56    while len(grouped) > 0:57        group, children = next(iter(grouped.items()))58        if not fs.exists(str(group)):59            raise ValueError(f"Path {group} does not exist")60        if (61            not group.parent in grouped or not group.name in grouped[group.parent]62        ) and not str(group) in non_collapsable:63            all_children_included = True64            _, dirs, files = next(fs.walk(str(group)))65            for path in dirs + files:66                path = Path(path.name)67                if path.parts[0] not in children and (68                    predicate is None or predicate(path)69                ):70                    all_children_included = False71                    break72            if all_children_included:73                if group.parent not in grouped:74                    grouped[group.parent] = set()75                grouped[group.parent].add(group.name)76            else:77                non_collapsable = non_collapsable.union(78                    [str((group / child)) for child in children]79                )80        del grouped[group]81    return sorted(non_collapsable)82def create_fs() -> FS:83    path = Path(".").resolve()84    current_path = path85    while True:86        lst = list(current_path.glob("bohr.py"))87        if len(lst) > 0 and lst[0].is_file():88            return OSFS(str(current_path))89        elif current_path == Path("/"):90            raise ValueError(91                f"Not a bohr directory: {path}. "92                f"Bohr config dir is not found in this or any parent directory"93            )94        else:95            current_path = current_path.parent96def gitignore_file(fs: FS, filename: str):97    """98    >>> from fs.memoryfs import MemoryFS99    >>> fs = MemoryFS()100    >>> gitignore_file(fs, 'file')101    >>> fs.readtext(".gitignore")102    'file\\n'103    >>> fs = MemoryFS()104    >>> fs.touch('file')105    >>> gitignore_file(fs, 'file')106    >>> fs.readtext(".gitignore")107    'file\\n'108    >>> gitignore_file(fs, 'file')109    >>> fs.readtext(".gitignore")110    'file\\n'111    >>> gitignore_file(fs, 'file2')112    >>> fs.readtext(".gitignore")113    'file\\nfile2\\n'114    """115    fs.touch(".gitignore")116    with fs.open(".gitignore", "r") as f:117        lines = list(map(lambda l: l.rstrip("\n"), f.readlines()))118        if filename not in lines:119            with fs.open(".gitignore", "a") as a:...

plugin_fs_hierarchy.py

Source:plugin_fs_hierarchy.py

1"""2<author>3  Savvas Savvides4<created>5  May 20136<description>7  Uses the parser to get a list of actions. Examines actions that contain a path8  and constructs the fs hierarchy of all files that exist.9  This program aims for simplicity, not efficiency.10  This is meant to be a quick plugin to demonstrate the parser's usability. This 11  is not meant to be an optimal implementation12"""13import os14import sys15import parser_truss_calls16import parser_strace_calls17def print_fs_hierarchy(hierarchy, depth, path):18  """19    <Purpose>20      A recursive functin used to print the given hierarchy21    <Arguments>22      hierarchy: A dictionary representing the hierarchy.23    <Returns>24      None25  """26  # first print current path part27  print "    " * (depth),28  print path29  30  children = []31  if path in hierarchy:32    children = hierarchy[path]33    children.sort()34  35  for child in children:36    print_fs_hierarchy(hierarchy, depth+1, child)37def construct_hierarchy(paths, start):38  """39    <Purpose>40      Given a list of paths, construct and return their hierarchy41    <Arguments>42      paths: A list of paths43    <Returns>44      hierarchy: a dictionary with the hierarchy.45  """46  hierarchy = {}47  # first add the root48  hierarchy[start] = []49  for path in paths:50    # get the parts of the path51    parts = path.strip("./").split("/")52    parts = filter(None, parts)53    for part_index in range(len(parts)):54      # get the preceding part55      if part_index == 0:56        pre = start57      else:58        pre = "/" + "/".join(parts[:part_index])59      60      current_path = "/" + "/".join(parts[:part_index+1])61      # add current path to the list of the preceding part62      if current_path not in hierarchy[pre]:63        hierarchy[pre].append(current_path)64      # create an entry for the current part65      if current_path not in hierarchy:66        hierarchy[current_path] = []67  return hierarchy68def extract_paths(actions):69  """70    <Purpose>71      Given a list of actions, it extracts all the absolute and relative paths72      from all the actions.73    <Arguments>74      actions: A list of actions from a parsed trace75    <Returns>76      absolute_paths: a list with all absolute paths extracted from the actions77      relative_paths: a list with all relative paths extracted from the actions78  """79  absolute_paths = []80  relative_paths = []81  actions_with_path = ['open', 'creat', 'statfs', 'access', 'stat', 82                        'link', 'unlink', 'chdir', 'rmdir', 'mkdir']83  for action in actions:84    # get the name of the syscall and remove the "_syscall" part at the end.85    action_name = action[0][:action[0].find("_syscall")]86    87    # we only care about actions containing paths88    if action_name not in actions_with_path:89      continue90    # we only care about paths that exist91    action_result = action[2]92    if action_result == (-1, 'ENOENT'):93      continue94    path = action[1][0]95    if path.startswith("/"):96      if path not in absolute_paths:97        absolute_paths.append(path)98    else:99      if path not in relative_paths:100        relative_paths.append(path)101    # get the second path of link102    if action_name == "link":103      path = action[1][1]104      if path.startswith("/"):105        if path not in absolute_paths:106          absolute_paths.append(path)107      else:108        if path not in relative_paths:109          relative_paths.append(path)110  return absolute_paths, relative_paths111if __name__ == "__main__":112  if len(sys.argv) != 2:113    raise Exception("Please provide: trace file")114  115  fh = open(sys.argv[1], "r")116  actions = parser_strace_calls.parse_trace(fh)117  118  absolute_paths, relative_paths = extract_paths(actions)119  absolute_hierarchy = construct_hierarchy(absolute_paths, "/")120  print_fs_hierarchy(absolute_hierarchy, depth=0, path="/")121  print122  relative_hierarchy = construct_hierarchy(relative_paths, ".")123  print_fs_hierarchy(relative_hierarchy, depth=0, path=".")...

data_handler.py

Source:data_handler.py

1import os2import torch3from torch.utils.data import Dataset4import numpy as np5class Data_Handler(Dataset):6    def __init__(self, dataset_path, subset, transform=None):7        self.subset = subset8        self.transform = transform9        10        # Load data absolute paths from the dataset file11        with open(dataset_path, 'rb') as f:12            self.absolute_paths = np.load(f, allow_pickle=True).item()13            14    def __len__(self):15        for subset_key in self.absolute_paths:16            if self.subset in subset_key:17                return len(self.absolute_paths[subset_key])18        return 019    def __getitem__(self, i):20        header = image = mask = []21        for subset_key in self.absolute_paths:22            if self.subset in subset_key:23                subset_file_paths = self.absolute_paths[subset_key]24                sample_file_paths = list(subset_file_paths[i].values())25                # Load header26                file_path = sample_file_paths[0]27                if (os.path.exists(file_path)):28                    with open(file_path, 'rb') as f:29                        header = np.load(f, allow_pickle=True)30                file_path = sample_file_paths[1]31                32                if (os.path.exists(file_path)):33                    with open(file_path, 'rb') as f:34                        image = np.load(f, allow_pickle=True)35                36                # Load mask37                if not self.subset == 'test':38                    file_path = sample_file_paths[2]39                    if (os.path.exists(file_path)):40                        with open(file_path, 'rb') as f:41                            mask = np.load(f, allow_pickle=True)42        sample = {43            'header': header.item(),44            'image': np.expand_dims(image, axis=0),45            'mask': np.expand_dims(mask, axis=0)46        }47        48        if self.transform is not None:49            sample = self.transform(sample)50            ...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.