How to use os_walk method in molecule

Best Python code snippet using molecule_python

search.py

Source:search.py Github

copy

Full Screen

1# -*- coding: utf-8 -*-2"""3Module with file search utilities.4"""5import os6import re7# -----------------------------------------------------------------------------8def find_files(root,9 prefix = None,10 suffix = None,11 dirname = None,12 direxcl = None):13 """14 Find files.15 ---16 type: function17 ...18 """19 if direxcl is None:20 direxcl = [r'^\..*$']21 pathincl_regex = r'^.*'22 if dirname is not None:23 pathincl_regex += dirname + os.sep24 if prefix is not None:25 prefix = prefix.replace('.', r'\.')26 pathincl_regex += prefix + r'.*'27 if suffix is not None:28 suffix = suffix.replace('.', r'\.')29 pathincl_regex += suffix30 pathincl_regex += r'$'31 pathincl = [pathincl_regex]32 return filtered_filepath_generator(33 root = root,34 direxcl = direxcl,35 pathincl = pathincl,36 pathexcl = None)37# -----------------------------------------------------------------------------38def filtered_dirpath_generator(root,39 direxcl = None,40 pathincl = None,41 pathexcl = None):42 """43 Return generator of dirpaths from root, filtered using regex lists.44 ---45 type: function46 ...47 """48 return _dirpath_from_os_walk_filter(49 os_walk = _dirname_filtered_os_walk_gen(root, direxcl = direxcl),50 pathincl = pathincl,51 pathexcl = pathexcl)52# -----------------------------------------------------------------------------53def filtered_filepath_generator(root,54 direxcl = None,55 pathincl = None,56 pathexcl = None):57 """58 Return generator of filepaths from root, filtered using regex lists.59 ---60 type: function61 ...62 """63 return _filepath_from_os_walk_filter(64 os_walk = _dirname_filtered_os_walk_gen(root, direxcl = direxcl),65 pathincl = pathincl,66 pathexcl = pathexcl)67# -----------------------------------------------------------------------------68def _dirpath_from_os_walk_filter(os_walk,69 pathincl = None,70 pathexcl = None):71 """72 Return filter of dirpaths, adapted to take an iterable of os.walk tuples.73 ---74 type: function75 ...76 """77 return _filepath_regex_filter(78 _adapt_os_walk_to_dirpath(os_walk), pathincl, pathexcl)79# -----------------------------------------------------------------------------80def _filepath_from_os_walk_filter(os_walk,81 pathincl = None,82 pathexcl = None):83 """84 Return filter of filepaths, adapted to take an iterable of os.walk tuples.85 ---86 type: function87 ...88 """89 return _filepath_regex_filter(90 _adapt_os_walk_to_filepath(os_walk), pathincl, pathexcl)91# -----------------------------------------------------------------------------92def _adapt_os_walk_to_dirpath(os_walk):93 """94 Return adapter converting os.walk tuple iterable into dirpath iterable.95 Intended to process the output of os_walk and96 dirname_filter functions.97 ---98 type: function99 ...100 """101 for (current_path, dir_list, _) in os_walk:102 for dir_name in dir_list:103 yield os.path.join(current_path, dir_name)104# -----------------------------------------------------------------------------105def _adapt_os_walk_to_filepath(os_walk):106 """107 Return adapter converting os.walk tuple iterable into filepath iterable.108 Intended to process the output of os_walk and109 dirname_filter functions.110 ---111 type: function112 ...113 """114 for (current_path, _, file_list) in os_walk:115 for file_name in sorted(file_list):116 yield os.path.join(current_path, file_name)117# -----------------------------------------------------------------------------118def _dirname_filtered_os_walk_gen(root,119 direxcl = None,120 onerror = None,121 followlinks = False):122 """123 Return generator of os.walk tuples, filtered using regex lists.124 ---125 type: function126 ...127 """128 return _dirname_regex_filter(129 os_walk = os.walk(root,130 topdown = True,131 onerror = onerror,132 followlinks = followlinks),133 excl = direxcl)134# -----------------------------------------------------------------------------135def _dirname_regex_filter(os_walk,136 excl = None):137 """138 Filter tuples generated by os.walk. Recursion limited by directory name.139 The supplied indicator function is used to140 decide if a directory subtree should be141 recursed into or not.142 ---143 type: function144 ...145 """146 dirname_indicator_func = get_dual_regex_indicator_fcn(excl = excl)147 return _dirname_filter(os_walk, dirname_indicator_func)148# -----------------------------------------------------------------------------149def _dirname_filter(os_walk, dirname_indicator_func):150 """151 Filter tuples generated by os.walk. Recursion limited by directory name.152 The supplied indicator function is used to153 decide if a directory subtree should be154 recursed into or not.155 ---156 type: function157 ...158 """159 for (current_path, subdir_list, file_list) in os_walk:160 if not subdir_list:161 subdir_list[:] = []162 else:163 subdir_list[:] = sorted(164 path for path in subdir_list if dirname_indicator_func(path))165 yield (current_path, subdir_list, file_list)166# -----------------------------------------------------------------------------167def _filepath_regex_filter(iter_filepaths,168 incl = None,169 excl = None):170 """171 Filter for filepaths, filtering specified using regex lists.172 ---173 type: function174 ...175 """176 filepath_indicator_func = get_dual_regex_indicator_fcn(incl, excl)177 return (path for path in iter_filepaths if filepath_indicator_func(path))178# -----------------------------------------------------------------------------179def get_dual_regex_indicator_fcn(incl=None, excl=None):180 """181 Indicator function for strings based on a pair of compiled regexes.182 - Returns True if incl matches and excl does not.183 - If incl is not specified or None, it always matches (always include).184 - If excl is not specified or None, it never matches (never exclude).185 ---186 type: function187 ...188 """189 is_incl = _get_regex_indicator_fcn(incl, default = True)190 is_excl = _get_regex_indicator_fcn(excl, default = False)191 return lambda item: is_incl(item) and not is_excl(item)192# -----------------------------------------------------------------------------193def _get_regex_indicator_fcn(regex_list=None, default=False):194 """195 Return an indicator function defined by the specified regular expression.196 ---197 type: function198 ...199 """200 if regex_list:201 regex = _compile_regex_list(regex_list)202 return lambda item: (regex.match(item) is not None)203 else:204 return lambda item: (default)205# -----------------------------------------------------------------------------206def _compile_regex_list(regex_list):207 """208 Compile a list of regex strings into a regular expression object.209 ---210 type: function211 ...212 """213 combined = "(" + ")|(".join(regex_list) + ")"214 compiled = re.compile(combined)215 return compiled216# -----------------------------------------------------------------------------217def find_ancestor_dir_containing(dir_path, marker_name, allow_dir=True):218 """219 Search for an ancestor directory of dir_path that contains a marker.220 This function identifies the closest (deepest)221 ancestor directory of dir_path that contains a222 file or directory named as specified by the223 marker_name parameter.224 It works by visiting each ancestor directory225 in dir_path in turn, starting at dir_path and226 proceeding up towards the root of the file-227 system hierarchy. At each step, it checks to228 see if a file or directory with the specified229 name exists. If it finds this marker, it returns230 the containing directory. If not, it continues231 towards the root. If, after reaching the root,232 no marker has yet been found, a SearchError233 exception is raised.234 This function was created to help identify235 the root of the local working copy, given a236 path within it. It should also be possible237 to use this function to help identify the238 root of various other filesystem hierarchies,239 given the emplacement of suitable marker files240 or directories.241 ---242 type: function243 ...244 """245 for dir_path in _walk_towards_root_generator(os.path.realpath(dir_path)):246 path_marker = os.path.join(dir_path, marker_name)247 is_file = os.path.isfile(path_marker)248 if allow_dir:249 is_dir = os.path.isdir(path_marker)250 is_marker = is_file or is_dir251 else:252 is_marker = is_file253 if is_marker:254 return dir_path255 raise RuntimeError("Could not find marker {name}".format(256 name = marker_name))257# -----------------------------------------------------------------------------258def _walk_towards_root_generator(dir_path):259 """260 Iterate over ancestor directories from dir_path up to the filesystem root.261 This function generates a sequence of directory262 paths starting with dir_path and progressively263 returning the parent directory of each until264 the filesystem root directory is reached.265 ---266 type: function267 ...268 """269 prev_path = None270 while dir_path != prev_path:271 yield dir_path272 prev_path = dir_path...

Full Screen

Full Screen

benchmark.py

Source:benchmark.py Github

copy

Full Screen

...60 raise betterwalk.posix_error(path)61 return names62else:63 raise NotImplementedError64def os_walk(top, topdown=True, onerror=None, followlinks=False):65 """Identical to os.walk(), but use ctypes-based listdir() so benchmark66 against ctypes-based iterdir_stat() is valid.67 """68 try:69 names = os_listdir(top)70 except OSError as err:71 if onerror is not None:72 onerror(err)73 return74 dirs, nondirs = [], []75 for name in names:76 if os.path.isdir(os.path.join(top, name)):77 dirs.append(name)78 else:79 nondirs.append(name)80 if topdown:81 yield top, dirs, nondirs82 for name in dirs:83 new_path = os.path.join(top, name)84 if followlinks or not os.path.islink(new_path):85 for x in os_walk(new_path, topdown, onerror, followlinks):86 yield x87 if not topdown:88 yield top, dirs, nondirs89def create_tree(path, depth=DEPTH):90 """Create a directory tree at path with given depth, and NUM_DIRS and91 NUM_FILES at each level.92 """93 os.mkdir(path)94 for i in range(NUM_FILES):95 filename = os.path.join(path, 'file{0:03}.txt'.format(i))96 with open(filename, 'wb') as f:97 line = b'The quick brown fox jumps over the lazy dog.\n'98 if i == 0:99 # So we have at least one big file per directory100 f.write(line * 20000)101 else:102 f.write(line * i * 10)103 if depth <= 1:104 return105 for i in range(NUM_DIRS):106 dirname = os.path.join(path, 'dir{0:03}'.format(i))107 create_tree(dirname, depth - 1)108def get_tree_size(path):109 """Return total size of all files in directory tree at path."""110 size = 0111 try:112 for name, st in betterwalk.iterdir_stat(path, fields=['st_mode_type', 'st_size']):113 if stat.S_ISDIR(st.st_mode):114 size += get_tree_size(os.path.join(path, name))115 else:116 size += st.st_size117 except OSError:118 pass119 return size120def benchmark(path, get_size=False):121 sizes = {}122 if get_size:123 def do_os_walk():124 size = 0125 for root, dirs, files in os_walk(path):126 for filename in files:127 fullname = os.path.join(root, filename)128 size += os.path.getsize(fullname)129 sizes['os_walk'] = size130 def do_betterwalk():131 sizes['betterwalk'] = get_tree_size(path)132 else:133 def do_os_walk():134 for root, dirs, files in os_walk(path):135 pass136 def do_betterwalk():137 for root, dirs, files in betterwalk.walk(path):138 pass139 # Run this once first to cache things, so we're not benchmarking I/O140 print("Priming the system's cache...")141 do_betterwalk()142 # Use the best of 3 time for each of them to eliminate high outliers143 os_walk_time = 1000000144 betterwalk_time = 1000000145 N = 3146 for i in range(N):147 print('Benchmarking walks on {0}, repeat {1}/{2}...'.format(148 path, i + 1, N))...

Full Screen

Full Screen

test_filemanager.py

Source:test_filemanager.py Github

copy

Full Screen

1""" Test the Filemanager module """2import os3import time4import pyres.filemanager5from mock import patch6class TestFilemanager(object):7 """ test the open functionality """8 def test_create(self, tmpdir):9 """ just call the cstor for the object """10 assert self11 filemgr = pyres.filemanager.FileManager(base_dir=tmpdir.dirname)12 assert filemgr.does_filesystem_exist()13 @patch('shutil.copyfile')14 def test_copy(self, mock_copy): # pylint: disable=W062115 """ copy files correctly """16 assert self17 episodes = [18 pyres.episode.Episode(19 date=time.localtime(),20 title='title',21 url='url',22 podcast='podcast',23 size=1234,24 state=1,25 # one of these two required26 base_path='base_path',27 file_name='file_name'28 ),29 pyres.episode.Episode(30 date=time.gmtime(), # use gm time to get a different value31 title='title',32 url='url',33 podcast='podcast',34 size=1234,35 state=1,36 # one of these two required37 base_path='base_path',38 file_name='file_name2'39 )40 ]41 copy_base = 'copy_base'42 filemgr = pyres.filemanager.FileManager(base_dir=copy_base)43 assert filemgr44 filemgr.copy_episodes_to_player(episodes)45 assert mock_copy.called46 assert mock_copy.call_count == 247 @patch('shutil.copyfile')48 @patch('pyres.filemanager.utils.mkdir_p')49 @patch('pyres.filemanager.os.walk')50 def test_audiobook_copy(self, os_walk, mkdir_p,51 copyfile): # pylint: disable=W062152 """ test simple file copy """53 assert self54 assert mkdir_p # to appease pylint55 os_walk.return_value = [56 ('/foo', ['bar', ], ('baz',)),57 ('/foo/bar', [], ('spam', 'eggs')),58 ]59 copy_base = 'copy_base'60 filemgr = pyres.filemanager.FileManager(base_dir=copy_base)61 filemgr.copy_audiobook("source")62 assert copyfile.called63 print(copyfile.call_args_list)64 assert copyfile.call_count == 365 assert copyfile.any_call('/foo/baz')66 assert copyfile.any_call('/foo/bar/spam')67 assert copyfile.any_call('/foo/bar/eggs')68 @patch('shutil.copyfile')69 @patch('pyres.filemanager.utils.mkdir_p')70 @patch('pyres.filemanager.os.walk')71 def test_audiobook_name(self, os_walk, mkdir_p,72 copyfile): # pylint: disable=W062173 """ test that names with numbers are converted correctly """74 assert self75 assert mkdir_p # to appease pylint76 def single_file(filename, expected):77 """ Test a single file """78 os_walk.return_value = [79 # if os.walk returns a root with leading /, os.path.join80 # doesn't add the filemanager base_dir to the path. This81 # makes it a bit easier to write these tests82 ('/foo', [], (filename,)),83 ]84 filemgr.copy_audiobook("source")85 copyfile.assert_called_with(os.path.join('/foo', filename),86 os.path.join('/foo', expected))87 copy_base = 'copy_base'88 filemgr = pyres.filemanager.FileManager(base_dir=copy_base)89 assert filemgr90 # one digit91 single_file('1one', '01one')92 single_file('o1ne', 'o01ne')93 single_file('one1', 'one01')94 # two digits95 single_file('11two', '11two')96 single_file('t11wo', 't11wo')97 single_file('two11', 'two11')98 # three digits99 single_file('123three', '123three')100 single_file('th123ree', 'th123ree')101 single_file('three123', 'three123')102 # mp3 extension not changed103 single_file('1baz.mp3', '01baz.mp3')104 single_file('b1az.mp3', 'b01az.mp3')105 single_file('baz1.mp3', 'baz01.mp3')106 # test multiple substitutions107 single_file('1b1a1z.1mp3', '01b01a01z.01mp3')108 @patch('shutil.copyfile')109 @patch('pyres.filemanager.utils.mkdir_p')110 @patch('pyres.filemanager.os.walk')111 def test_audiobook_dest_dir(self, os_walk, mkdir_p,112 copyfile): # pylint: disable=W0621113 """ test that the dest dir parameters is used correctly """114 assert self115 assert mkdir_p # to appease pylint116 copy_base = 'copy_base'117 filemgr = pyres.filemanager.FileManager(base_dir=copy_base)118 assert filemgr119 os_walk.return_value = [120 ('source', [], ("wilma",)),121 ]122 filemgr.copy_audiobook("source", "dest")123 print(copyfile.call_args_list)124 copyfile.assert_called_with(os.path.join('source', 'wilma'),125 os.path.join('copy_base', 'dest', 'source',126 'wilma'))127 @patch('shutil.copyfile')128 @patch('pyres.filemanager.utils.mkdir_p')129 @patch('pyres.filemanager.os.walk')130 def test_audiobook_deep_dir(self, os_walk, mkdir_p,131 copyfile): # pylint: disable=W0621132 """ test that directory structures are preserved """133 assert self134 assert mkdir_p # to appease pylint135 copy_base = 'copy_base'136 filemgr = pyres.filemanager.FileManager(base_dir=copy_base)137 assert filemgr138 os_walk.return_value = [139 # see note above about leading slashes on top level directory140 ('/first', ['second', ], ("wilma",)),141 ('/first/second', ['third', ], ("wilma",)),142 ('/first/second/third', ['', ], ("wilma", "fred")),143 ]144 filemgr.copy_audiobook("source")145 copyfile.assert_any_call(os.path.join('/first', 'wilma'),146 os.path.join('/first', 'wilma'))147 copyfile.assert_any_call(os.path.join('/first/second', 'wilma'),148 os.path.join('/first/second', 'wilma'))149 copyfile.assert_any_call(os.path.join('/first/second/third', 'wilma'),150 os.path.join('/first/second/third', 'wilma'))151 copyfile.assert_any_call(os.path.join('/first/second/third', 'fred'),...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run molecule automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful