How to use iter_file_paths method in prospector

Best Python code snippet using prospector_python

tokenization.py

Source:tokenization.py

...48        init_logger(LOGGER_NAME)49        inp_folder = self.input()[0]50        failed_file_count = 051        logger.info("Tokenizing `%s`." % inp_folder.path)52        for file_path in iter_file_paths(inp_folder.path):53            file_name = get_file_name(file_path)54            out_file_path = comb_paths(self.act_out_dir_path,55                                       "%s.csv" % file_name)56            try:57                # TODO: `_csv.Error: line contains NULL byte` was encountered in58                # TODO: a small number of files; the cause needs to be59                # TODO: investigated60                TokenizeFile(inp_file_path=file_path,61                             out_file_path=out_file_path).run()62            except CsvError:63                failed_file_count += 164        logger.info('Failed to tokenize `%d` files in `%s`.' %65                    (failed_file_count, inp_folder.path))66class TokenizeFile(Task):...

doc2txt.py

Source:doc2txt.py

1import os2import time3import subprocess4from concurrent.futures import ThreadPoolExecutor, as_completed, ProcessPoolExecutor5def iter_file_paths(path, size):6    file_infos = []7    for file in os.listdir(path):8        file_infos.append(dict(9            file_name=''.join(file.split('.')[:-1]),10            file_path=os.path.join(path, file)11        ))12        if all([file_infos, len(file_infos) % size == 0]):13            yield file_infos14            file_infos = []15    if file_infos:16        yield file_infos17def doc_txt(file_info):18    file_path = file_info['file_path']19    out = subprocess.check_output('antiword {}'.format(file_path), shell=True)20    return out.decode()21def run(path):22    start = time.time()23    data = []24    with ProcessPoolExecutor() as executor:25        for file_infos in iter_file_paths(path=path, size=100):26            futures = [executor.submit(doc_txt, file_info) for file_info in file_infos]27            for future in as_completed(futures):28                error = future.exception()29                data.append(future.result())30                if time.time() - start >= 1:31                    print(len(data))32                    return33if __name__ == '__main__':...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.