Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use copy_files_from method in autotest

Best Python code snippet using autotest_python

standard.py

Source:standard.py

1from collections import OrderedDict2import os3from copy import deepcopy4from mlcomp.contrib.search.grid import grid_cells5from mlcomp.db.core import Session6from mlcomp.db.enums import TaskType, DagType, ComponentType7from mlcomp.db.models import Report, Task, Dag, ReportTasks, TaskDependence8from mlcomp.db.providers import TaskProvider, \9    ReportProvider, \10    ReportTasksProvider, \11    ReportLayoutProvider, \12    DagProvider, \13    ProjectProvider14from mlcomp.utils.misc import now15from mlcomp.worker.executors import Executor16from mlcomp.worker.storage import Storage17from mlcomp.utils.io import yaml_dump18class DagStandardBuilder:19    def __init__(20            self,21            session: Session,22            config: dict,23            debug: bool,24            config_text: str = None,25            upload_files: bool = True,26            copy_files_from: int = None,27            config_path: str = None,28            control_reqs: bool = True,29            logger=None,30            component: ComponentType = None,31            grid_cell: dict = None32    ):33        self.session = session34        self.config = config35        self.debug = debug36        self.config_text = config_text37        self.upload_files = upload_files38        self.copy_files_from = copy_files_from39        self.config_path = config_path40        self.control_reqs = control_reqs41        self.info = config['info']42        self.layout_name = self.info.get('layout')43        self.provider = None44        self.report_provider = None45        self.report_tasks_provider = None46        self.report_layout_provider = None47        self.storage = None48        self.dag_provider = None49        self.logger = logger50        self.component = component51        self.grid_cell = grid_cell52        self.project = None53        self.layouts = None54        self.dag = None55        self.dag_report_id = None56        self.created = None57        self.project_provider = None58    def log_info(self, message: str):59        if self.logger:60            self.logger.info(message, self.component)61    def create_providers(self):62        self.log_info('create_providers')63        self.provider = TaskProvider(self.session)64        self.report_provider = ReportProvider(self.session)65        self.report_tasks_provider = ReportTasksProvider(self.session)66        self.report_layout_provider = ReportLayoutProvider(self.session)67        self.project_provider = ProjectProvider(self.session)68        self.storage = Storage(self.session, logger=self.logger,69                               component=self.component)70        self.dag_provider = DagProvider(self.session)71    def load_base(self):72        self.log_info('load_base')73        project = self.project_provider.by_name(self.info['project'])74        if project is None:75            project = self.project_provider.add_project(self.info['project'])76        self.project = project.id77        self.layouts = self.report_layout_provider.all()78    def create_report(self):79        self.log_info('create_report')80        self.dag_report_id = None81        layout_name = self.layout_name82        if layout_name:83            if layout_name not in self.layouts:84                raise Exception(f'Unknown layout = {layout_name}')85            report = Report(86                config=yaml_dump(self.layouts[layout_name]),87                name=self.info['name'],88                project=self.project,89                layout=layout_name90            )91            self.report_provider.add(report)92            self.dag_report_id = report.id93    def create_dag(self):94        self.log_info('create_dag')95        name = self.info['name']96        if self.grid_cell:97            name = f'{name} {self.grid_cell[1]}'98        dag = Dag(99            config=self.config_text or yaml_dump(self.config),100            project=self.project,101            name=name,102            docker_img=self.info.get('docker_img'),103            type=DagType.Standard.value,104            created=now(),105            report=self.dag_report_id106        )107        self.dag = self.dag_provider.add(dag)108    def upload(self):109        self.log_info('upload')110        if self.upload_files:111            folder = os.path.dirname(os.path.abspath(self.config_path))112            if 'expdir' in self.config['info']:113                path = os.path.dirname(os.path.abspath(self.config_path))114                folder = os.path.abspath(115                    os.path.join(path, self.config['info']['expdir'])116                )117            self.storage.upload(folder, self.dag,118                                control_reqs=self.control_reqs)119        elif self.copy_files_from:120            self.storage.copy_from(self.copy_files_from, self.dag)121    def create_task(self, k: str, v: dict, name: str, info: dict,122                    cell: dict = None):123        task_type = TaskType.User.value124        v = deepcopy(v)125        if v.get('task_type') == 'train' or \126                Executor.is_trainable(v['type']):127            task_type = TaskType.Train.value128        gpu = str(v.get('gpu', '0'))129        if '-' not in gpu:130            gpu = int(gpu)131            gpu_max = gpu132        else:133            gpu, gpu_max = map(int, gpu.split('-'))134        if gpu == 0 and gpu_max > 0:135            raise Exception(f'Executor = {k} Gpu_max can"t be>0 when gpu=0')136        task = Task(137            name=name,138            executor=k,139            computer=self.info.get('computer') or v.get('computer'),140            gpu=gpu,141            gpu_max=gpu_max,142            cpu=v.get('cpu', 1),143            memory=v.get('memory', 0.1),144            dag=self.dag.id,145            debug=self.debug,146            steps=int(v.get('steps', '1')),147            type=task_type148        )149        if cell is not None:150            v.update(cell)151        info['executor'] = v152        task.additional_info = yaml_dump(info)153        report = None154        if self.layout_name and task_type == TaskType.Train.value:155            if self.layout_name not in self.layouts:156                raise Exception(f'Unknown report = {v["report"]}')157            report_config = self.layouts[self.layout_name]158            info['report_config'] = report_config159            task.additional_info = yaml_dump(info)160            report = Report(161                config=yaml_dump(report_config),162                name=task.name,163                project=self.project,164                layout=self.layout_name165            )166        return task, report167    def create_tasks(self):168        self.log_info('create_tasks')169        created = OrderedDict()170        executors = self.config['executors']171        tasks = []172        dependencies = []173        reports = []174        while len(created) < len(executors):175            for k, v in executors.items():176                if self.grid_cell:177                    v.update(**self.grid_cell[0])178                valid = True179                if 'depends' in v:180                    depends = v['depends']181                    if not isinstance(depends, list):182                        depends = [depends]183                    for d in depends:184                        if d == k:185                            raise Exception(f'Executor {k} depends on itself')186                        if d not in executors:187                            raise Exception(188                                f'Executor {k} depend on {d} '189                                f'which does not exist'190                            )191                        valid = valid and d in created192                if valid:193                    names = []194                    infos = []195                    task_cells = []196                    if 'grid' in v:197                        grid = v['grid']198                        del v['grid']199                        cells = grid_cells(grid)200                        for i, (cell, cell_name) in enumerate(cells):201                            names.append(cell_name)202                            infos.append({'grid_cell': i})203                            task_cells.append(cell)204                    else:205                        names.append(v.get('name', k))206                        infos.append({})207                        task_cells.append({})208                    k_tasks = []209                    for name, cell, info in zip(names, task_cells, infos):210                        task, report = self.create_task(k, v, name=name,211                                                        info=info, cell=cell)212                        tasks.append(task)213                        k_tasks.append(task)214                        reports.append(report)215                        if 'depends' in v:216                            depends = v['depends']217                            if not isinstance(depends, list):218                                depends = [depends]219                            for d in depends:220                                for dd in created[d]:221                                    dependencies.append((task, dd))222                    created[k] = k_tasks223        not_empty_reports = [r for r in reports if r is not None]224        if len(not_empty_reports) > 0:225            self.provider.bulk_save_objects(not_empty_reports,226                                            return_defaults=True)227            for report, task in zip(reports, tasks):228                if report is not None:229                    task.report = report.id230        self.provider.bulk_save_objects(tasks,231                                        return_defaults=True)232        if len(not_empty_reports) > 0:233            report_tasks = []234            for report, task in zip(reports, tasks):235                if report is not None:236                    report_tasks.append(237                        ReportTasks(report=report.id, task=task.id))238            self.report_tasks_provider.bulk_save_objects(report_tasks)239        dependencies = [240            TaskDependence(task_id=task.id, depend_id=dd.id) for task, dd in241            dependencies242        ]243        self.provider.bulk_save_objects(dependencies)244        for k, v in created.items():245            created[k] = [vv.id for vv in v]246        self.created = created247    def build(self):248        self.create_providers()249        self.load_base()250        self.create_report()251        self.create_dag()252        self.upload()253        self.create_tasks()254        self.log_info('Done')255        return self.created256def dag_standard(257        session: Session,258        config: dict,259        debug: bool,260        config_text: str = None,261        upload_files: bool = True,262        copy_files_from: int = None,263        config_path: str = None,264        control_reqs: bool = True,265        logger=None,266        component: ComponentType = None,267        grid_cell: dict = None268):269    builder = DagStandardBuilder(270        session=session,271        config=config,272        debug=debug,273        config_text=config_text,274        upload_files=upload_files,275        copy_files_from=copy_files_from,276        config_path=config_path,277        control_reqs=control_reqs,278        logger=logger,279        component=component,280        grid_cell=grid_cell281    )282    return builder.build()...

ethtool.py

Source:ethtool.py

...110                           lambda:session.cmd_status("pgrep tcpdump") == 0, 30):111            return (False, "Tcpdump process wasn't launched")112        logging.info("Start to transfer file")113        try:114            copy_files_from(filename, filename)115        except virt_utils.SCPError, e:116            return (False, "File transfer failed (%s)" % e)117        logging.info("Transfer file completed")118        session.cmd("killall tcpdump")119        try:120            tcpdump_string = session2.read_up_to_prompt(timeout=60)121        except aexpect.ExpectError:122            return (False, "Fail to read tcpdump's output")123        if not compare_md5sum(filename):124            return (False, "Files' md5sum mismatched")125        return (True, tcpdump_string)126    def tx_callback(status="on"):127        s, o = transfer_file(src="guest")128        if not s:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.