How to use copy_files_from method in autotest

Best Python code snippet using autotest_python Github


Full Screen

1from collections import OrderedDict2import os3from copy import deepcopy4from import grid_cells5from mlcomp.db.core import Session6from mlcomp.db.enums import TaskType, DagType, ComponentType7from mlcomp.db.models import Report, Task, Dag, ReportTasks, TaskDependence8from mlcomp.db.providers import TaskProvider, \9 ReportProvider, \10 ReportTasksProvider, \11 ReportLayoutProvider, \12 DagProvider, \13 ProjectProvider14from mlcomp.utils.misc import now15from mlcomp.worker.executors import Executor16from import Storage17from import yaml_dump18class DagStandardBuilder:19 def __init__(20 self,21 session: Session,22 config: dict,23 debug: bool,24 config_text: str = None,25 upload_files: bool = True,26 copy_files_from: int = None,27 config_path: str = None,28 control_reqs: bool = True,29 logger=None,30 component: ComponentType = None,31 grid_cell: dict = None32 ):33 self.session = session34 self.config = config35 self.debug = debug36 self.config_text = config_text37 self.upload_files = upload_files38 self.copy_files_from = copy_files_from39 self.config_path = config_path40 self.control_reqs = control_reqs41 = config['info']42 self.layout_name ='layout')43 self.provider = None44 self.report_provider = None45 self.report_tasks_provider = None46 self.report_layout_provider = None47 = None48 self.dag_provider = None49 self.logger = logger50 self.component = component51 self.grid_cell = grid_cell52 self.project = None53 self.layouts = None54 self.dag = None55 self.dag_report_id = None56 self.created = None57 self.project_provider = None58 def log_info(self, message: str):59 if self.logger:60, self.component)61 def create_providers(self):62 self.log_info('create_providers')63 self.provider = TaskProvider(self.session)64 self.report_provider = ReportProvider(self.session)65 self.report_tasks_provider = ReportTasksProvider(self.session)66 self.report_layout_provider = ReportLayoutProvider(self.session)67 self.project_provider = ProjectProvider(self.session)68 = Storage(self.session, logger=self.logger,69 component=self.component)70 self.dag_provider = DagProvider(self.session)71 def load_base(self):72 self.log_info('load_base')73 project = self.project_provider.by_name(['project'])74 if project is None:75 project = self.project_provider.add_project(['project'])76 self.project = project.id77 self.layouts = self.report_layout_provider.all()78 def create_report(self):79 self.log_info('create_report')80 self.dag_report_id = None81 layout_name = self.layout_name82 if layout_name:83 if layout_name not in self.layouts:84 raise Exception(f'Unknown layout = {layout_name}')85 report = Report(86 config=yaml_dump(self.layouts[layout_name]),87['name'],88 project=self.project,89 layout=layout_name90 )91 self.report_provider.add(report)92 self.dag_report_id = report.id93 def create_dag(self):94 self.log_info('create_dag')95 name =['name']96 if self.grid_cell:97 name = f'{name} {self.grid_cell[1]}'98 dag = Dag(99 config=self.config_text or yaml_dump(self.config),100 project=self.project,101 name=name,102'docker_img'),103 type=DagType.Standard.value,104 created=now(),105 report=self.dag_report_id106 )107 self.dag = self.dag_provider.add(dag)108 def upload(self):109 self.log_info('upload')110 if self.upload_files:111 folder = os.path.dirname(os.path.abspath(self.config_path))112 if 'expdir' in self.config['info']:113 path = os.path.dirname(os.path.abspath(self.config_path))114 folder = os.path.abspath(115 os.path.join(path, self.config['info']['expdir'])116 )117, self.dag,118 control_reqs=self.control_reqs)119 elif self.copy_files_from:120, self.dag)121 def create_task(self, k: str, v: dict, name: str, info: dict,122 cell: dict = None):123 task_type = TaskType.User.value124 v = deepcopy(v)125 if v.get('task_type') == 'train' or \126 Executor.is_trainable(v['type']):127 task_type = TaskType.Train.value128 gpu = str(v.get('gpu', '0'))129 if '-' not in gpu:130 gpu = int(gpu)131 gpu_max = gpu132 else:133 gpu, gpu_max = map(int, gpu.split('-'))134 if gpu == 0 and gpu_max > 0:135 raise Exception(f'Executor = {k} Gpu_max can"t be>0 when gpu=0')136 task = Task(137 name=name,138 executor=k,139'computer') or v.get('computer'),140 gpu=gpu,141 gpu_max=gpu_max,142 cpu=v.get('cpu', 1),143 memory=v.get('memory', 0.1),144,145 debug=self.debug,146 steps=int(v.get('steps', '1')),147 type=task_type148 )149 if cell is not None:150 v.update(cell)151 info['executor'] = v152 task.additional_info = yaml_dump(info)153 report = None154 if self.layout_name and task_type == TaskType.Train.value:155 if self.layout_name not in self.layouts:156 raise Exception(f'Unknown report = {v["report"]}')157 report_config = self.layouts[self.layout_name]158 info['report_config'] = report_config159 task.additional_info = yaml_dump(info)160 report = Report(161 config=yaml_dump(report_config),162,163 project=self.project,164 layout=self.layout_name165 )166 return task, report167 def create_tasks(self):168 self.log_info('create_tasks')169 created = OrderedDict()170 executors = self.config['executors']171 tasks = []172 dependencies = []173 reports = []174 while len(created) < len(executors):175 for k, v in executors.items():176 if self.grid_cell:177 v.update(**self.grid_cell[0])178 valid = True179 if 'depends' in v:180 depends = v['depends']181 if not isinstance(depends, list):182 depends = [depends]183 for d in depends:184 if d == k:185 raise Exception(f'Executor {k} depends on itself')186 if d not in executors:187 raise Exception(188 f'Executor {k} depend on {d} '189 f'which does not exist'190 )191 valid = valid and d in created192 if valid:193 names = []194 infos = []195 task_cells = []196 if 'grid' in v:197 grid = v['grid']198 del v['grid']199 cells = grid_cells(grid)200 for i, (cell, cell_name) in enumerate(cells):201 names.append(cell_name)202 infos.append({'grid_cell': i})203 task_cells.append(cell)204 else:205 names.append(v.get('name', k))206 infos.append({})207 task_cells.append({})208 k_tasks = []209 for name, cell, info in zip(names, task_cells, infos):210 task, report = self.create_task(k, v, name=name,211 info=info, cell=cell)212 tasks.append(task)213 k_tasks.append(task)214 reports.append(report)215 if 'depends' in v:216 depends = v['depends']217 if not isinstance(depends, list):218 depends = [depends]219 for d in depends:220 for dd in created[d]:221 dependencies.append((task, dd))222 created[k] = k_tasks223 not_empty_reports = [r for r in reports if r is not None]224 if len(not_empty_reports) > 0:225 self.provider.bulk_save_objects(not_empty_reports,226 return_defaults=True)227 for report, task in zip(reports, tasks):228 if report is not None:229 = report.id230 self.provider.bulk_save_objects(tasks,231 return_defaults=True)232 if len(not_empty_reports) > 0:233 report_tasks = []234 for report, task in zip(reports, tasks):235 if report is not None:236 report_tasks.append(237 ReportTasks(, self.report_tasks_provider.bulk_save_objects(report_tasks)239 dependencies = [240 TaskDependence(, for task, dd in241 dependencies242 ]243 self.provider.bulk_save_objects(dependencies)244 for k, v in created.items():245 created[k] = [ for vv in v]246 self.created = created247 def build(self):248 self.create_providers()249 self.load_base()250 self.create_report()251 self.create_dag()252 self.upload()253 self.create_tasks()254 self.log_info('Done')255 return self.created256def dag_standard(257 session: Session,258 config: dict,259 debug: bool,260 config_text: str = None,261 upload_files: bool = True,262 copy_files_from: int = None,263 config_path: str = None,264 control_reqs: bool = True,265 logger=None,266 component: ComponentType = None,267 grid_cell: dict = None268):269 builder = DagStandardBuilder(270 session=session,271 config=config,272 debug=debug,273 config_text=config_text,274 upload_files=upload_files,275 copy_files_from=copy_files_from,276 config_path=config_path,277 control_reqs=control_reqs,278 logger=logger,279 component=component,280 grid_cell=grid_cell281 )282 return

Full Screen

Full Screen Github


Full Screen

...110 lambda:session.cmd_status("pgrep tcpdump") == 0, 30):111 return (False, "Tcpdump process wasn't launched")112"Start to transfer file")113 try:114 copy_files_from(filename, filename)115 except virt_utils.SCPError, e:116 return (False, "File transfer failed (%s)" % e)117"Transfer file completed")118 session.cmd("killall tcpdump")119 try:120 tcpdump_string = session2.read_up_to_prompt(timeout=60)121 except aexpect.ExpectError:122 return (False, "Fail to read tcpdump's output")123 if not compare_md5sum(filename):124 return (False, "Files' md5sum mismatched")125 return (True, tcpdump_string)126 def tx_callback(status="on"):127 s, o = transfer_file(src="guest")128 if not s:...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:


You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run autotest automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?