How to use run_suite method in Lemoncheesecake

Best Python code snippet using lemoncheesecake

run_suite_unittest.py

Source:run_suite_unittest.py Github

copy

Full Screen

1#!/usr/bin/python2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.3# Use of this source code is governed by a BSD-style license that can be4# found in the LICENSE file.5import datetime as datetime_base6from datetime import datetime7import mock8import time9import unittest10import common11from autotest_lib.server.cros.dynamic_suite import constants12from autotest_lib.site_utils import run_suite13from autotest_lib.site_utils import diagnosis_utils14class ResultCollectorUnittest(unittest.TestCase):15 """Runsuite unittest"""16 JOB_MAX_RUNTIME_MINS = 1017 def setUp(self):18 """Set up test."""19 self.afe = mock.MagicMock()20 self.tko = mock.MagicMock()21 def _build_view(self, test_idx, test_name, subdir, status, afe_job_id,22 job_name='fake_job_name', reason='fake reason',23 job_keyvals=None, test_started_time=None,24 test_finished_time=None, invalidates_test_idx=None,25 job_started_time=None, job_finished_time=None):26 """Build a test view using the given fields.27 @param test_idx: An integer representing test_idx.28 @param test_name: A string, e.g. 'dummy_Pass'29 @param subdir: A string representing the subdir field of the test view.30 e.g. 'dummy_Pass'.31 @param status: A string representing the test status.32 e.g. 'FAIL', 'PASS'33 @param afe_job_id: An integer representing the afe job id.34 @param job_name: A string representing the job name.35 @param reason: A string representing the reason field of the test view.36 @param job_keyvals: A dictionary stroing the job keyvals.37 @param test_started_time: A string, e.g. '2014-04-12 12:35:33'38 @param test_finished_time: A string, e.g. '2014-04-12 12:35:33'39 @param invalidates_test_idx: An integer, representing the idx of the40 test that has been retried.41 @param job_started_time: A string, e.g. '2014-04-12 12:35:33'42 @param job_finished_time: A string, e.g. '2014-04-12 12:35:33'43 @reutrn: A dictionary representing a test view.44 """45 if job_keyvals is None:46 job_keyvals = {}47 return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir,48 'status': status, 'afe_job_id': afe_job_id,49 'job_name': job_name, 'reason': reason,50 'job_keyvals': job_keyvals,51 'test_started_time': test_started_time,52 'test_finished_time': test_finished_time,53 'invalidates_test_idx': invalidates_test_idx,54 'job_started_time': job_started_time,55 'job_finished_time': job_finished_time}56 def _mock_tko_get_detailed_test_views(self, test_views,57 missing_results=[]):58 """Mock tko method get_detailed_test_views call.59 @param test_views: A list of test views that will be returned60 by get_detailed_test_views.61 """62 return_values = {}63 for v in test_views:64 views_of_job = return_values.setdefault(65 ('get_detailed_test_views', v['afe_job_id']), [])66 views_of_job.append(v)67 for job_id in missing_results:68 views_of_job = return_values.setdefault(69 ('get_detailed_test_views', job_id), [])70 def side_effect(*args, **kwargs):71 """Maps args and kwargs to the mocked return values."""72 key = (kwargs['call'], kwargs['afe_job_id'])73 return return_values[key]74 self.tko.run = mock.MagicMock(side_effect=side_effect)75 def _mock_afe_get_jobs(self, suite_job_id, child_job_ids):76 """Mock afe get_jobs call.77 @param suite_job_id: The afe job id of the suite job.78 @param child_job_ids: A list of job ids of the child jobs.79 """80 suite_job = mock.MagicMock()81 suite_job.id = suite_job_id82 suite_job.max_runtime_mins = 1083 suite_job.parent_job = None84 return_values = {suite_job_id: []}85 for job_id in child_job_ids:86 new_job = mock.MagicMock()87 new_job.id = job_id88 new_job.name = 'test.%d' % job_id89 new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS90 new_job.parent_job = suite_job91 return_values[suite_job_id].append(new_job)92 def side_effect(*args, **kwargs):93 """Maps args and kwargs to the mocked return values."""94 if kwargs.get('id') == suite_job_id:95 return [suite_job]96 return return_values[kwargs['parent_job_id']]97 self.afe.get_jobs = mock.MagicMock(side_effect=side_effect)98 def testFetchSuiteTestView(self):99 """Test that it fetches the correct suite test views."""100 suite_job_id = 100101 suite_name = 'dummy'102 build = 'R23-1.1.1.1'103 server_job_view = self._build_view(104 10, 'SERVER_JOB', '----', 'GOOD', suite_job_id)105 test_to_ignore = self._build_view(106 11, 'dummy_Pass', '101-user/host/dummy_Pass',107 'GOOD', suite_job_id)108 test_to_include = self._build_view(109 12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id)110 test_missing = self._build_view(111 13, 'dummy_Missing', None, 'ABORT', suite_job_id)112 self._mock_afe_get_jobs(suite_job_id, [])113 self._mock_tko_get_detailed_test_views(114 [server_job_view, test_to_ignore, test_to_include,115 test_missing])116 collector = run_suite.ResultCollector(117 'fake_server', self.afe, self.tko,118 build='fake/build', board='fake', suite_name='dummy',119 suite_job_id=suite_job_id,120 return_code_function=run_suite._ReturnCodeComputer())121 collector._missing_results = {122 test_missing['test_name']: [14, 15],123 }124 suite_views = collector._fetch_relevant_test_views_of_suite()125 suite_views = sorted(suite_views, key=lambda view: view['test_idx'])126 # Verify that SERVER_JOB is renamed to 'Suite job'127 self.assertEqual(suite_views[0].get_testname(),128 run_suite.TestView.SUITE_JOB)129 # Verify that the test with a subidr is not included.130 self.assertEqual(suite_views[0]['test_idx'], 10)131 self.assertEqual(suite_views[1]['test_idx'], 12)132 self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id)133 # Verify that the test with missing results had it's AFE job id134 # replaced.135 self.assertEqual(suite_views[2]['test_idx'], 13)136 self.assertEqual(suite_views[2]['afe_job_id'], 14)137 def testFetchTestViewOfChildJobs(self):138 """Test that it fetches the correct child test views."""139 build = 'lumpy-release/R36-5788.0.0'140 board = 'lumpy'141 suite_name = 'my_suite'142 suite_job_id = 100143 invalid_job_id = 101144 invalid_job_name = '%s/%s/test_Pass' % (build, suite_name)145 good_job_id = 102146 good_job_name = '%s/%s/test_Pass' % (build, suite_name)147 bad_job_id = 103148 bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name)149 missing_job_id = 104150 invalid_test = self._build_view(151 19, 'test_Pass_Old', 'fake/subdir',152 'FAIL', invalid_job_id, invalid_job_name)153 good_job_server_job = self._build_view(154 20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name)155 good_job_test = self._build_view(156 21, 'test_Pass', 'fake/subdir', 'GOOD',157 good_job_id, good_job_name,158 job_keyvals={'retry_original_job_id': invalid_job_id})159 bad_job_server_job = self._build_view(160 22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name)161 bad_job_test = self._build_view(162 23, 'test_ServerJobFail', 'fake/subdir', 'GOOD',163 bad_job_id, bad_job_name)164 self._mock_tko_get_detailed_test_views(165 [good_job_server_job, good_job_test,166 bad_job_server_job, bad_job_test, invalid_test],167 [missing_job_id])168 self._mock_afe_get_jobs(suite_job_id,169 [good_job_id, bad_job_id, missing_job_id])170 collector = run_suite.ResultCollector(171 'fake_server', self.afe, self.tko,172 build, board, suite_name, suite_job_id,173 return_code_function=run_suite._ReturnCodeComputer())174 child_views, retry_counts, missing_results = (175 collector._fetch_test_views_of_child_jobs())176 # child_views should contain tests 21, 22, 23177 child_views = sorted(child_views, key=lambda view: view['test_idx'])178 # Verify that the SERVER_JOB has been renamed properly179 self.assertEqual(child_views[1].get_testname(),180 'test_ServerJobFail_SERVER_JOB')181 self.assertEqual(missing_results, {'test.104': [104]})182 # Verify that failed SERVER_JOB and actual invalid tests are included,183 expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'],184 bad_job_test['test_idx']]185 child_view_ids = [v['test_idx'] for v in child_views]186 self.assertEqual(child_view_ids, expected)187 self.afe.get_jobs.assert_called_once_with(188 parent_job_id=suite_job_id)189 # Verify the retry_counts is calculated correctly190 self.assertEqual(len(retry_counts), 1)191 self.assertEqual(retry_counts[21], 1)192 def testGenerateLinks(self):193 """Test that it generates correct web and buildbot links."""194 suite_job_id = 100195 suite_name = 'my_suite'196 build = 'lumpy-release/R36-5788.0.0'197 board = 'lumpy'198 fake_job = mock.MagicMock()199 fake_job.parent = suite_job_id200 suite_job_view = run_suite.TestView(201 self._build_view(202 20, 'Suite job', '----', 'GOOD', suite_job_id),203 fake_job, suite_name, build, 'chromeos-test')204 good_test = run_suite.TestView(205 self._build_view(206 21, 'test_Pass', 'fake/subdir', 'GOOD', 101),207 fake_job, suite_name, build, 'chromeos-test')208 bad_test = run_suite.TestView(209 self._build_view(210 23, 'test_Fail', 'fake/subdir', 'FAIL', 102),211 fake_job, suite_name, build, 'chromeos-test')212 collector = run_suite.ResultCollector(213 'fake_server', self.afe, self.tko,214 build, board, suite_name, suite_job_id, user='chromeos-test',215 return_code_function=run_suite._ReturnCodeComputer())216 collector._suite_views = [suite_job_view]217 collector._test_views = [suite_job_view, good_test, bad_test]218 collector._max_testname_width = max(219 [len(v.get_testname()) for v in collector._test_views]) + 3220 collector._generate_web_and_buildbot_links()221 URL_PATTERN = run_suite._URL_PATTERN222 # expected_web_links is list of (anchor, url) tuples we223 # are expecting.224 expected_web_links = [225 (v.get_testname(),226 URL_PATTERN % ('fake_server',227 '%s-%s' % (v['afe_job_id'], 'chromeos-test')))228 for v in collector._test_views]229 # Verify web links are generated correctly.230 for i in range(len(collector._web_links)):231 expect = expected_web_links[i]232 self.assertEqual(collector._web_links[i].anchor, expect[0])233 self.assertEqual(collector._web_links[i].url, expect[1])234 expected_buildbot_links = [235 (v.get_testname(),236 URL_PATTERN % ('fake_server',237 '%s-%s' % (v['afe_job_id'], 'chromeos-test')))238 for v in collector._test_views if v['status'] != 'GOOD']239 # Verify buildbot links are generated correctly.240 for i in range(len(collector.buildbot_links)):241 expect = expected_buildbot_links[i]242 self.assertEqual(collector.buildbot_links[i].anchor, expect[0])243 self.assertEqual(collector.buildbot_links[i].url, expect[1])244 self.assertEqual(collector.buildbot_links[i].retry_count, 0)245 # Assert that a wmatrix retry dashboard link is created.246 self.assertNotEqual(247 collector.buildbot_links[i].GenerateWmatrixRetryLink(), '')248 self.assertNotEqual(249 collector.buildbot_links[i].GenerateWmatrixHistoryLink(),250 '')251 def _end_to_end_test_helper(252 self, include_bad_test=False, include_warn_test=False,253 include_timeout_test=False,254 include_self_aborted_test=False,255 include_aborted_by_suite_test=False,256 include_good_retry=False, include_bad_retry=False,257 suite_job_timed_out=False, suite_job_status='GOOD'):258 """A helper method for testing ResultCollector end-to-end.259 This method mocks the retrieving of required test views,260 and call ResultCollector.run() to collect the results.261 @param include_bad_test:262 If True, include a view of a test which has status 'FAIL'.263 @param include_warn_test:264 If True, include a view of a test which has status 'WARN'265 @param include_timeout_test:266 If True, include a view of a test which was aborted before267 started.268 @param include_self_aborted_test:269 If True, include a view of test which was aborted after270 started and hit hits own timeout.271 @param include_self_aborted_by_suite_test:272 If True, include a view of test which was aborted after273 started but has not hit its own timeout.274 @param include_good_retry:275 If True, include a test that passed after retry.276 @param include_bad_retry:277 If True, include a test that failed after retry.278 @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING'279 @returns: A ResultCollector instance.280 """281 suite_job_id = 100282 good_job_id = 101283 bad_job_id = 102284 warn_job_id = 102285 timeout_job_id = 100286 self_aborted_job_id = 104287 aborted_by_suite_job_id = 105288 good_retry_job_id = 106289 bad_retry_job_id = 107290 invalid_job_id_1 = 90291 invalid_job_id_2 = 91292 suite_name = 'dummy'293 build = 'lumpy-release/R27-3888.0.0'294 suite_job_keyvals = {295 constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20',296 constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25',297 constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'}298 suite_job_started_time = '2014-04-29 13:14:37'299 if suite_job_timed_out:300 suite_job_keyvals['aborted_by'] = 'test_user'301 suite_job_finished_time = '2014-04-29 13:25:37'302 suite_job_status = 'ABORT'303 else:304 suite_job_finished_time = '2014-04-29 13:23:37'305 server_job_view = self._build_view(306 10, 'SERVER_JOB', '----', suite_job_status, suite_job_id,307 'lumpy-release/R27-3888.0.0-test_suites/control.dummy',308 '', suite_job_keyvals, '2014-04-29 13:14:37',309 '2014-04-29 13:20:27', job_started_time=suite_job_started_time,310 job_finished_time=suite_job_finished_time)311 good_test = self._build_view(312 11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD',313 good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass',314 '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36')315 bad_test = self._build_view(316 12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL',317 bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail',318 'always fail', {}, '2014-04-29 13:16:00',319 '2014-04-29 13:16:02')320 warn_test = self._build_view(321 13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN',322 warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn',323 'always warn', {}, '2014-04-29 13:16:00',324 '2014-04-29 13:16:02')325 timeout_test = self._build_view(326 15, 'dummy_Timeout', '', 'ABORT',327 timeout_job_id,328 'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout',329 'child job did not run', {}, '2014-04-29 13:15:37',330 '2014-04-29 13:15:38')331 self_aborted_test = self._build_view(332 16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT',333 self_aborted_job_id,334 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',335 'child job aborted', {'aborted_by': 'test_user'},336 '2014-04-29 13:15:39', '2014-04-29 13:15:40',337 job_started_time='2014-04-29 13:15:39',338 job_finished_time='2014-04-29 13:25:40')339 aborted_by_suite = self._build_view(340 17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite',341 'RUNNING', aborted_by_suite_job_id,342 'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',343 'aborted by suite', {'aborted_by': 'test_user'},344 '2014-04-29 13:15:39', '2014-04-29 13:15:40',345 job_started_time='2014-04-29 13:15:39',346 job_finished_time='2014-04-29 13:15:40')347 good_retry = self._build_view(348 18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD',349 good_retry_job_id,350 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',351 '', {'retry_original_job_id': invalid_job_id_1},352 '2014-04-29 13:15:37',353 '2014-04-29 13:15:38', invalidates_test_idx=1)354 bad_retry = self._build_view(355 19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL',356 bad_retry_job_id,357 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',358 'retry failed', {'retry_original_job_id': invalid_job_id_2},359 '2014-04-29 13:15:39', '2014-04-29 13:15:40',360 invalidates_test_idx=2)361 invalid_test_1 = self._build_view(362 1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD',363 invalid_job_id_1,364 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',365 'original test failed', {}, '2014-04-29 13:10:00',366 '2014-04-29 13:10:01')367 invalid_test_2 = self._build_view(368 2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL',369 invalid_job_id_2,370 'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',371 'original test failed', {},372 '2014-04-29 13:10:03', '2014-04-29 13:10:04')373 test_views = [server_job_view, good_test]374 child_jobs = set([good_job_id])375 if include_bad_test:376 test_views.append(bad_test)377 child_jobs.add(bad_job_id)378 if include_warn_test:379 test_views.append(warn_test)380 child_jobs.add(warn_job_id)381 if include_timeout_test:382 test_views.append(timeout_test)383 if include_self_aborted_test:384 test_views.append(self_aborted_test)385 child_jobs.add(self_aborted_job_id)386 if include_good_retry:387 test_views.extend([good_retry, invalid_test_1])388 child_jobs.add(good_retry_job_id)389 if include_bad_retry:390 test_views.extend([bad_retry, invalid_test_2])391 child_jobs.add(bad_retry_job_id)392 if include_aborted_by_suite_test:393 test_views.append(aborted_by_suite)394 child_jobs.add(aborted_by_suite_job_id)395 self._mock_tko_get_detailed_test_views(test_views)396 self._mock_afe_get_jobs(suite_job_id, child_jobs)397 collector = run_suite.ResultCollector(398 'fake_server', self.afe, self.tko,399 'lumpy-release/R36-5788.0.0', 'lumpy', 'dummy', suite_job_id,400 return_code_function=run_suite._ReturnCodeComputer())401 collector.run()402 return collector403 def testEndToEndSuitePass(self):404 """Test it returns code OK when all test pass."""405 collector = self._end_to_end_test_helper()406 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.OK)407 def testEndToEndSuiteWarn(self):408 """Test it returns code WARNING when there is a test that warns."""409 collector = self._end_to_end_test_helper(include_warn_test=True)410 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.WARNING)411 def testEndToEndSuiteFail(self):412 """Test it returns code ERROR when there is a test that fails."""413 collector = self._end_to_end_test_helper(include_bad_test=True)414 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR)415 def testEndToEndSuiteJobFail(self):416 """Test it returns code SUITE_FAILURE when only the suite job failed."""417 collector = self._end_to_end_test_helper(suite_job_status='ABORT')418 self.assertEqual(419 collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE)420 collector = self._end_to_end_test_helper(suite_job_status='ERROR')421 self.assertEqual(422 collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE)423 def testEndToEndRetry(self):424 """Test it returns correct code when a test was retried."""425 collector = self._end_to_end_test_helper(include_good_retry=True)426 self.assertEqual(427 collector.return_code, run_suite.RETURN_CODES.WARNING)428 collector = self._end_to_end_test_helper(include_good_retry=True,429 include_self_aborted_test=True)430 self.assertEqual(431 collector.return_code, run_suite.RETURN_CODES.ERROR)432 collector = self._end_to_end_test_helper(include_good_retry=True,433 include_bad_test=True)434 self.assertEqual(435 collector.return_code, run_suite.RETURN_CODES.ERROR)436 collector = self._end_to_end_test_helper(include_bad_retry=True)437 self.assertEqual(438 collector.return_code, run_suite.RETURN_CODES.ERROR)439 def testEndToEndSuiteTimeout(self):440 """Test it returns correct code when a child job timed out."""441 # a child job timed out before started, none failed.442 collector = self._end_to_end_test_helper(include_timeout_test=True)443 self.assertEqual(444 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)445 # a child job timed out before started, and one test failed.446 collector = self._end_to_end_test_helper(447 include_bad_test=True, include_timeout_test=True)448 self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR)449 # a child job timed out before started, and one test warned.450 collector = self._end_to_end_test_helper(451 include_warn_test=True, include_timeout_test=True)452 self.assertEqual(collector.return_code,453 run_suite.RETURN_CODES.SUITE_TIMEOUT)454 # a child job timed out before started, and one test was retried.455 collector = self._end_to_end_test_helper(include_good_retry=True,456 include_timeout_test=True)457 self.assertEqual(458 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)459 # a child jot was aborted because suite timed out.460 collector = self._end_to_end_test_helper(461 include_aborted_by_suite_test=True)462 self.assertEqual(463 collector.return_code, run_suite.RETURN_CODES.OK)464 # suite job timed out.465 collector = self._end_to_end_test_helper(suite_job_timed_out=True)466 self.assertEqual(467 collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)468class LogLinkUnittests(unittest.TestCase):469 """Test the LogLink"""470 def testGenerateBuildbotLinks(self):471 """Test LogLink GenerateBuildbotLinks"""472 log_link_a = run_suite.LogLink('mock_anchor', 'mock_server',473 'mock_job_string',474 bug_info=('mock_bug_id', 1),475 reason='mock_reason',476 retry_count=1,477 testname='mock_testname')478 # Generate a bug link and a log link when bug_info is present479 self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2)480 log_link_b = run_suite.LogLink('mock_anchor', 'mock_server',481 'mock_job_string_b',482 reason='mock_reason',483 retry_count=1,484 testname='mock_testname')485 # Generate a log link when there is no bug_info486 self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1)487class SimpleTimerUnittests(unittest.TestCase):488 """Test the simple timer."""489 def testPoll(self):490 """Test polling the timer."""491 interval_hours = 0.0001492 t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours)493 deadline = t.deadline494 self.assertTrue(deadline is not None and495 t.interval_hours == interval_hours)496 min_deadline = (datetime.now() +497 datetime_base.timedelta(hours=interval_hours))498 time.sleep(interval_hours * 3600)499 self.assertTrue(t.poll())500 self.assertTrue(t.deadline >= min_deadline)501 def testBadInterval(self):502 """Test a bad interval."""503 t = diagnosis_utils.SimpleTimer(interval_hours=-1)504 self.assertTrue(t.deadline is None and t.poll() == False)505 t._reset()506 self.assertTrue(t.deadline is None and t.poll() == False)507class ArgumentParserUnittests(unittest.TestCase):508 """Tests for argument parser."""509 @unittest.expectedFailure510 def test_crbug_658013(self):511 """crbug.com/658013512 Expected failure due to http://bugs.python.org/issue9334513 """514 parser = run_suite.make_parser()515 args = [516 '--board', 'heli',517 '--build', 'trybot-heli-paladin/R56-8918.0.0-b1601',518 '--suite_name', 'test_that_wrapper',519 '--pool', 'suites',520 '--max_runtime_mins', '20',521 '--suite_args', '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline',522 ]523 def error_handler(msg): # pylint: disable=missing-docstring524 self.fail('Argument parsing failed: ' + msg)525 parser.error = error_handler526 got = parser.parse_args(args)527 self.assertEqual(528 got.board, 'heli')529 self.assertEqual(530 got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601')531 self.assertEqual(532 got.suite_args,533 '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline')534 def test_crbug_658013b(self):535 """crbug.com/658013536 Unambiguous behavior.537 """538 parser = run_suite.make_parser()539 args = [540 '--board=heli',541 '--build=trybot-heli-paladin/R56-8918.0.0-b1601',542 '--suite_name=test_that_wrapper',543 '--pool=suites',544 '--max_runtime_mins=20',545 '--suite_args=-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline',546 ]547 def error_handler(msg): # pylint: disable=missing-docstring548 self.fail('Argument parsing failed: ' + msg)549 parser.error = error_handler550 got = parser.parse_args(args)551 self.assertEqual(552 got.board, 'heli')553 self.assertEqual(554 got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601')555 self.assertEqual(556 got.suite_args,557 '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline')558if __name__ == '__main__':...

Full Screen

Full Screen

test_event_order.py

Source:test_event_order.py Github

copy

Full Screen

...41 self.event_order.append(test_skipped)42 def on_test_started(self, **kwargs):43 self.event_order.append(test_started)44 def test_happy_path(self):45 def run_suite():46 class Suite(TestSuite):47 def test_nothing(self):48 pass49 run_suite()50 expect(self.event_order).to(equal([suite_started, test_started, test_ended, suite_ended]))51 def test_suite_setup_error(self):52 def run_suite():53 class Suite(TestSuite):54 def setup_suite(self):55 raise RuntimeError("intentional")56 def test_nothing(self):57 pass58 run_suite()59 expect(self.event_order).to(equal([suite_started, suite_erred, test_skipped, suite_ended]))60 def test_test_setup_error(self):61 def run_suite():62 class Suite(TestSuite):63 def setup(self):64 raise RuntimeError("intentional")65 def test_should_not_run(self):66 pass67 run_suite()68 expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))69 def test_test_error(self):70 def run_suite():71 class Suite(TestSuite):72 def test_go_boom(self):73 raise RuntimeError("intentional")74 run_suite()75 expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))76 def test_test_failure(self):77 def run_suite():78 class Suite(TestSuite):79 def test_fails(self):80 raise AssertionError("intentional")81 run_suite()82 expect(self.event_order).to(equal([suite_started, test_started, test_failed, test_ended, suite_ended]))83 def test_test_skip(self):84 def run_suite():85 class Suite(TestSuite):86 def test_fails(self):87 raise TestSkipped("intentional")88 run_suite()89 expect(self.event_order).to(equal([suite_started, test_started, test_skipped, test_ended, suite_ended]))90 def test_test_teardown_failure(self):91 def run_suite():92 class Suite(TestSuite):93 def test_nothing(self):94 pass95 def teardown(self):96 raise RuntimeError("intentional")97 run_suite()98 expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))99 def test_suite_teardown_failure(self):100 def run_suite():101 class Suite(TestSuite):102 def test_nothing(self):103 pass104 def teardown_suite(self):105 raise RuntimeError("intentional")106 run_suite()107 expect(self.event_order).to(equal([suite_started, test_started, test_ended, suite_erred, suite_ended]))108if "__main__" == __name__:...

Full Screen

Full Screen

runners.py

Source:runners.py Github

copy

Full Screen

...39 except Exception, e:40 errors.append(f[0])41 traceback.print_exc()42 return passed, failed, errors43def run_suite(modules, smoke=False, report=False):44 passed = []45 failed = []46 errors = []47 target = 048 try:49 git_dir = os.path.dirname(os.path.dirname(__file__))50 git_sha1 = ave.git.rev_list(git_dir, 1)[0]51 except Exception, e:52 git_sha1 = ''53 for m in modules:54 args = tuple()55 if type(m) == tuple:56 args = m[1]57 m = m[0]58 package = 'ave.common.%s' % m.__name__.split('.')[-1]59 functions = inspect.getmembers(m, inspect.isfunction)60 functions = [f for f in functions if re.match('t\d+', f[0])]61 if smoke:62 functions = [f for f in functions if hasattr(f[1], 'smoke')]63 target += len(functions)64 p,f,e = run_tests(functions, package, *args)65 passed.extend(p)66 failed.extend(f)67 errors.extend(e)68 if len(passed) != target:69 return vcsjob.FAILURES70 return vcsjob.OK71def all_smoke(report, so_path):72 modules = [73 tests.config, tests.cmd, tests.documentation, tests.spool,74 tests.connection, tests.control, (tests.fdtx, (so_path,))75 ]76 return run_suite(modules, True, report)77def all_config(report=False):78 return run_suite([tests.config], False, report)79def all_cmd(report=False):80 return run_suite([tests.cmd], False, report)81def all_documentation(report=False):82 return run_suite([tests.documentation], False, report)83def all_spool(report=False):84 # tests for irrelevant functionality. don't run85 return run_suite([tests.spool], False, report)86def all_connection(report=False):87 return run_suite([tests.connection], False, report)88def all_control(report=False):89 run_suite([tests.control_sync], False, report)90 run_suite([tests.control_async], False, report)91# for i in range(100000):92# tests.control_async.t28()93# print 'num fds: %d' % len(os.listdir('/proc/%d/fd' % os.getpid()))94# gc.collect()95# pass96def all_fdtx(so_path=None, report=False):97 return run_suite([(tests.fdtx, (so_path,))], False, report)98def all_exception(report=False):99 return run_suite([tests.exception], False, report)100def all_hickup(report=False):101 return run_suite([tests.hickup], False, report)102def all_stress():103 return tests.stress.t1()104def all_panotti(report=False):105 return run_suite([tests.panotti], False, report)106def all_process(report=False):107 return run_suite([tests.process], False, report)108def all_pipe(report=False):109 return run_suite([tests.pipe], False, report)110def all_daemon(report=False):111 return run_suite([tests.daemon], False, report)112def all_defjoin(report=False):113 return run_suite([tests.defjoin], False, report)114def all_async_rpc(report=False):...

Full Screen

Full Screen

test_gpu_updaters.py

Source:test_gpu_updaters.py Github

copy

Full Screen

...16 def test_gpu_exact(self):17 variable_param = {'max_depth': [2, 6, 15], }18 for param in parameter_combinations(variable_param):19 param['tree_method'] = 'gpu_exact'20 gpu_results = run_suite(param, select_datasets=datasets)21 assert_results_non_increasing(gpu_results, 1e-2)22 param['tree_method'] = 'exact'23 cpu_results = run_suite(param, select_datasets=datasets)24 assert_gpu_results(cpu_results, gpu_results)25 def test_gpu_hist(self):26 test_param = parameter_combinations({'n_gpus': [1], 'max_depth': [2, 8],27 'max_leaves': [255, 4],28 'max_bin': [2, 256],29 'grow_policy': ['lossguide']})30 test_param.append({'single_precision_histogram': True})31 test_param.append({'min_child_weight': 0,32 'lambda': 0})33 for param in test_param:34 param['tree_method'] = 'gpu_hist'35 gpu_results = run_suite(param, select_datasets=datasets)36 assert_results_non_increasing(gpu_results, 1e-2)37 param['tree_method'] = 'hist'38 cpu_results = run_suite(param, select_datasets=datasets)39 assert_gpu_results(cpu_results, gpu_results)40 @pytest.mark.mgpu41 def test_gpu_hist_mgpu(self):42 variable_param = {'n_gpus': [-1], 'max_depth': [2, 10],43 'max_leaves': [255, 4],44 'max_bin': [2, 256],45 'grow_policy': ['lossguide'], 'debug_synchronize': [True]}46 for param in parameter_combinations(variable_param):47 param['tree_method'] = 'gpu_hist'48 gpu_results = run_suite(param, select_datasets=datasets)49 assert_results_non_increasing(gpu_results, 1e-2)50 @pytest.mark.mgpu51 def test_specified_gpu_id_gpu_update(self):52 variable_param = {'n_gpus': [1],53 'gpu_id': [1],54 'max_depth': [8],55 'max_leaves': [255, 4],56 'max_bin': [2, 64],57 'grow_policy': ['lossguide'],58 'tree_method': ['gpu_hist', 'gpu_exact']}59 for param in parameter_combinations(variable_param):60 gpu_results = run_suite(param, select_datasets=datasets)...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run Lemoncheesecake automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful