Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use run_suite method in Lemoncheesecake

Best Python code snippet using lemoncheesecake

run_suite_unittest.py

Source:run_suite_unittest.py

1#!/usr/bin/python2# Copyright (c) 2014 The Chromium OS Authors. All rights reserved.3# Use of this source code is governed by a BSD-style license that can be4# found in the LICENSE file.5import datetime as datetime_base6from datetime import datetime7import mock8import time9import unittest10import common11from autotest_lib.server.cros.dynamic_suite import constants12from autotest_lib.site_utils import run_suite13from autotest_lib.site_utils import diagnosis_utils14class ResultCollectorUnittest(unittest.TestCase):15    """Runsuite unittest"""16    JOB_MAX_RUNTIME_MINS = 1017    def setUp(self):18        """Set up test."""19        self.afe = mock.MagicMock()20        self.tko = mock.MagicMock()21    def _build_view(self, test_idx, test_name, subdir, status, afe_job_id,22                    job_name='fake_job_name', reason='fake reason',23                    job_keyvals=None, test_started_time=None,24                    test_finished_time=None, invalidates_test_idx=None,25                    job_started_time=None, job_finished_time=None):26        """Build a test view using the given fields.27        @param test_idx: An integer representing test_idx.28        @param test_name: A string, e.g. 'dummy_Pass'29        @param subdir: A string representing the subdir field of the test view.30                       e.g. 'dummy_Pass'.31        @param status: A string representing the test status.32                       e.g. 'FAIL', 'PASS'33        @param afe_job_id: An integer representing the afe job id.34        @param job_name: A string representing the job name.35        @param reason: A string representing the reason field of the test view.36        @param job_keyvals: A dictionary stroing the job keyvals.37        @param test_started_time: A string, e.g. '2014-04-12 12:35:33'38        @param test_finished_time: A string, e.g. '2014-04-12 12:35:33'39        @param invalidates_test_idx: An integer, representing the idx of the40                                     test that has been retried.41        @param job_started_time: A string, e.g. '2014-04-12 12:35:33'42        @param job_finished_time: A string, e.g. '2014-04-12 12:35:33'43        @reutrn: A dictionary representing a test view.44        """45        if job_keyvals is None:46            job_keyvals = {}47        return {'test_idx': test_idx, 'test_name': test_name, 'subdir':subdir,48                'status': status, 'afe_job_id': afe_job_id,49                'job_name': job_name, 'reason': reason,50                'job_keyvals': job_keyvals,51                'test_started_time': test_started_time,52                'test_finished_time': test_finished_time,53                'invalidates_test_idx': invalidates_test_idx,54                'job_started_time': job_started_time,55                'job_finished_time': job_finished_time}56    def _mock_tko_get_detailed_test_views(self, test_views,57                                          missing_results=[]):58        """Mock tko method get_detailed_test_views call.59        @param test_views: A list of test views that will be returned60                           by get_detailed_test_views.61        """62        return_values = {}63        for v in test_views:64            views_of_job = return_values.setdefault(65                    ('get_detailed_test_views', v['afe_job_id']), [])66            views_of_job.append(v)67        for job_id in missing_results:68            views_of_job = return_values.setdefault(69                    ('get_detailed_test_views', job_id), [])70        def side_effect(*args, **kwargs):71            """Maps args and kwargs to the mocked return values."""72            key = (kwargs['call'], kwargs['afe_job_id'])73            return return_values[key]74        self.tko.run = mock.MagicMock(side_effect=side_effect)75    def _mock_afe_get_jobs(self, suite_job_id, child_job_ids):76        """Mock afe get_jobs call.77        @param suite_job_id: The afe job id of the suite job.78        @param child_job_ids: A list of job ids of the child jobs.79        """80        suite_job = mock.MagicMock()81        suite_job.id = suite_job_id82        suite_job.max_runtime_mins = 1083        suite_job.parent_job = None84        return_values = {suite_job_id: []}85        for job_id in child_job_ids:86            new_job = mock.MagicMock()87            new_job.id = job_id88            new_job.name = 'test.%d' % job_id89            new_job.max_runtime_mins = self.JOB_MAX_RUNTIME_MINS90            new_job.parent_job = suite_job91            return_values[suite_job_id].append(new_job)92        def side_effect(*args, **kwargs):93            """Maps args and kwargs to the mocked return values."""94            if kwargs.get('id') == suite_job_id:95                return [suite_job]96            return return_values[kwargs['parent_job_id']]97        self.afe.get_jobs = mock.MagicMock(side_effect=side_effect)98    def testFetchSuiteTestView(self):99        """Test that it fetches the correct suite test views."""100        suite_job_id = 100101        suite_name = 'dummy'102        build = 'R23-1.1.1.1'103        server_job_view = self._build_view(104                10, 'SERVER_JOB', '----', 'GOOD', suite_job_id)105        test_to_ignore = self._build_view(106                11, 'dummy_Pass', '101-user/host/dummy_Pass',107                'GOOD', suite_job_id)108        test_to_include = self._build_view(109                12, 'dummy_Pass.bluetooth', None, 'TEST_NA', suite_job_id)110        test_missing = self._build_view(111                13, 'dummy_Missing', None, 'ABORT', suite_job_id)112        self._mock_afe_get_jobs(suite_job_id, [])113        self._mock_tko_get_detailed_test_views(114                [server_job_view, test_to_ignore, test_to_include,115                 test_missing])116        collector = run_suite.ResultCollector(117                'fake_server', self.afe, self.tko,118                build='fake/build', board='fake', suite_name='dummy',119                suite_job_id=suite_job_id,120                return_code_function=run_suite._ReturnCodeComputer())121        collector._missing_results = {122                test_missing['test_name']: [14, 15],123        }124        suite_views = collector._fetch_relevant_test_views_of_suite()125        suite_views = sorted(suite_views, key=lambda view: view['test_idx'])126        # Verify that SERVER_JOB is renamed to 'Suite job'127        self.assertEqual(suite_views[0].get_testname(),128                         run_suite.TestView.SUITE_JOB)129        # Verify that the test with a subidr is not included.130        self.assertEqual(suite_views[0]['test_idx'], 10)131        self.assertEqual(suite_views[1]['test_idx'], 12)132        self.assertEqual(suite_views[1]['afe_job_id'], suite_job_id)133        # Verify that the test with missing results had it's AFE job id134        # replaced.135        self.assertEqual(suite_views[2]['test_idx'], 13)136        self.assertEqual(suite_views[2]['afe_job_id'], 14)137    def testFetchTestViewOfChildJobs(self):138        """Test that it fetches the correct child test views."""139        build = 'lumpy-release/R36-5788.0.0'140        board = 'lumpy'141        suite_name = 'my_suite'142        suite_job_id = 100143        invalid_job_id = 101144        invalid_job_name = '%s/%s/test_Pass' % (build, suite_name)145        good_job_id = 102146        good_job_name = '%s/%s/test_Pass' % (build, suite_name)147        bad_job_id = 103148        bad_job_name = '%s/%s/test_ServerJobFail' % (build, suite_name)149        missing_job_id = 104150        invalid_test = self._build_view(151                19, 'test_Pass_Old', 'fake/subdir',152                'FAIL', invalid_job_id, invalid_job_name)153        good_job_server_job = self._build_view(154                20, 'SERVER_JOB', '----', 'GOOD', good_job_id, good_job_name)155        good_job_test = self._build_view(156                21, 'test_Pass', 'fake/subdir', 'GOOD',157                good_job_id, good_job_name,158                job_keyvals={'retry_original_job_id': invalid_job_id})159        bad_job_server_job = self._build_view(160                22, 'SERVER_JOB', '----', 'FAIL', bad_job_id, bad_job_name)161        bad_job_test = self._build_view(162                23, 'test_ServerJobFail', 'fake/subdir', 'GOOD',163                bad_job_id, bad_job_name)164        self._mock_tko_get_detailed_test_views(165                [good_job_server_job, good_job_test,166                 bad_job_server_job, bad_job_test, invalid_test],167                [missing_job_id])168        self._mock_afe_get_jobs(suite_job_id,169                                [good_job_id, bad_job_id, missing_job_id])170        collector = run_suite.ResultCollector(171                'fake_server', self.afe, self.tko,172                build, board, suite_name, suite_job_id,173                return_code_function=run_suite._ReturnCodeComputer())174        child_views, retry_counts, missing_results = (175                collector._fetch_test_views_of_child_jobs())176        # child_views should contain tests 21, 22, 23177        child_views = sorted(child_views, key=lambda view: view['test_idx'])178        # Verify that the SERVER_JOB has been renamed properly179        self.assertEqual(child_views[1].get_testname(),180                         'test_ServerJobFail_SERVER_JOB')181        self.assertEqual(missing_results, {'test.104': [104]})182        # Verify that failed SERVER_JOB and actual invalid tests are included,183        expected = [good_job_test['test_idx'], bad_job_server_job['test_idx'],184                    bad_job_test['test_idx']]185        child_view_ids = [v['test_idx'] for v in child_views]186        self.assertEqual(child_view_ids, expected)187        self.afe.get_jobs.assert_called_once_with(188                parent_job_id=suite_job_id)189        # Verify the retry_counts is calculated correctly190        self.assertEqual(len(retry_counts), 1)191        self.assertEqual(retry_counts[21], 1)192    def testGenerateLinks(self):193        """Test that it generates correct web and buildbot links."""194        suite_job_id = 100195        suite_name = 'my_suite'196        build = 'lumpy-release/R36-5788.0.0'197        board = 'lumpy'198        fake_job = mock.MagicMock()199        fake_job.parent = suite_job_id200        suite_job_view = run_suite.TestView(201                self._build_view(202                    20, 'Suite job', '----', 'GOOD', suite_job_id),203                fake_job, suite_name, build, 'chromeos-test')204        good_test = run_suite.TestView(205                self._build_view(206                    21, 'test_Pass', 'fake/subdir', 'GOOD', 101),207                fake_job, suite_name, build, 'chromeos-test')208        bad_test = run_suite.TestView(209                self._build_view(210                    23, 'test_Fail', 'fake/subdir', 'FAIL', 102),211                fake_job, suite_name, build, 'chromeos-test')212        collector = run_suite.ResultCollector(213                'fake_server', self.afe, self.tko,214                build, board, suite_name, suite_job_id, user='chromeos-test',215                return_code_function=run_suite._ReturnCodeComputer())216        collector._suite_views = [suite_job_view]217        collector._test_views = [suite_job_view, good_test, bad_test]218        collector._max_testname_width = max(219                [len(v.get_testname()) for v in collector._test_views]) + 3220        collector._generate_web_and_buildbot_links()221        URL_PATTERN = run_suite._URL_PATTERN222        # expected_web_links is list of (anchor, url) tuples we223        # are expecting.224        expected_web_links = [225                 (v.get_testname(),226                  URL_PATTERN % ('fake_server',227                                '%s-%s' % (v['afe_job_id'], 'chromeos-test')))228                 for v in collector._test_views]229        # Verify web links are generated correctly.230        for i in range(len(collector._web_links)):231            expect = expected_web_links[i]232            self.assertEqual(collector._web_links[i].anchor, expect[0])233            self.assertEqual(collector._web_links[i].url, expect[1])234        expected_buildbot_links = [235                 (v.get_testname(),236                  URL_PATTERN % ('fake_server',237                                '%s-%s' % (v['afe_job_id'], 'chromeos-test')))238                 for v in collector._test_views if v['status'] != 'GOOD']239        # Verify buildbot links are generated correctly.240        for i in range(len(collector.buildbot_links)):241            expect = expected_buildbot_links[i]242            self.assertEqual(collector.buildbot_links[i].anchor, expect[0])243            self.assertEqual(collector.buildbot_links[i].url, expect[1])244            self.assertEqual(collector.buildbot_links[i].retry_count, 0)245            # Assert that a wmatrix retry dashboard link is created.246            self.assertNotEqual(247                    collector.buildbot_links[i].GenerateWmatrixRetryLink(), '')248            self.assertNotEqual(249                    collector.buildbot_links[i].GenerateWmatrixHistoryLink(),250                    '')251    def _end_to_end_test_helper(252            self, include_bad_test=False, include_warn_test=False,253            include_timeout_test=False,254            include_self_aborted_test=False,255            include_aborted_by_suite_test=False,256            include_good_retry=False, include_bad_retry=False,257            suite_job_timed_out=False, suite_job_status='GOOD'):258        """A helper method for testing ResultCollector end-to-end.259        This method mocks the retrieving of required test views,260        and call ResultCollector.run() to collect the results.261        @param include_bad_test:262                If True, include a view of a test which has status 'FAIL'.263        @param include_warn_test:264                If True, include a view of a test which has status 'WARN'265        @param include_timeout_test:266                If True, include a view of a test which was aborted before267                started.268        @param include_self_aborted_test:269                If True, include a view of test which was aborted after270                started and hit hits own timeout.271        @param include_self_aborted_by_suite_test:272                If True, include a view of test which was aborted after273                started but has not hit its own timeout.274        @param include_good_retry:275                If True, include a test that passed after retry.276        @param include_bad_retry:277                If True, include a test that failed after retry.278        @param suite_job_status: One of 'GOOD' 'FAIL' 'ABORT' 'RUNNING'279        @returns: A ResultCollector instance.280        """281        suite_job_id = 100282        good_job_id = 101283        bad_job_id = 102284        warn_job_id = 102285        timeout_job_id = 100286        self_aborted_job_id = 104287        aborted_by_suite_job_id = 105288        good_retry_job_id = 106289        bad_retry_job_id = 107290        invalid_job_id_1 = 90291        invalid_job_id_2 = 91292        suite_name = 'dummy'293        build = 'lumpy-release/R27-3888.0.0'294        suite_job_keyvals = {295                constants.DOWNLOAD_STARTED_TIME: '2014-04-29 13:14:20',296                constants.PAYLOAD_FINISHED_TIME: '2014-04-29 13:14:25',297                constants.ARTIFACT_FINISHED_TIME: '2014-04-29 13:14:30'}298        suite_job_started_time = '2014-04-29 13:14:37'299        if suite_job_timed_out:300            suite_job_keyvals['aborted_by'] = 'test_user'301            suite_job_finished_time = '2014-04-29 13:25:37'302            suite_job_status = 'ABORT'303        else:304            suite_job_finished_time = '2014-04-29 13:23:37'305        server_job_view = self._build_view(306                10, 'SERVER_JOB', '----', suite_job_status, suite_job_id,307                'lumpy-release/R27-3888.0.0-test_suites/control.dummy',308                '', suite_job_keyvals, '2014-04-29 13:14:37',309                '2014-04-29 13:20:27', job_started_time=suite_job_started_time,310                job_finished_time=suite_job_finished_time)311        good_test = self._build_view(312                11, 'dummy_Pass', '101-user/host/dummy_Pass', 'GOOD',313                good_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Pass',314                '', {}, '2014-04-29 13:15:35', '2014-04-29 13:15:36')315        bad_test = self._build_view(316                12, 'dummy_Fail.Fail', '102-user/host/dummy_Fail.Fail', 'FAIL',317                bad_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Fail',318                'always fail', {}, '2014-04-29 13:16:00',319                '2014-04-29 13:16:02')320        warn_test = self._build_view(321                13, 'dummy_Fail.Warn', '102-user/host/dummy_Fail.Warn', 'WARN',322                warn_job_id, 'lumpy-release/R27-3888.0.0/dummy/dummy_Fail.Warn',323                'always warn', {}, '2014-04-29 13:16:00',324                '2014-04-29 13:16:02')325        timeout_test = self._build_view(326                15, 'dummy_Timeout', '', 'ABORT',327                timeout_job_id,328                'lumpy-release/R27-3888.0.0/dummy/dummy_Timeout',329                'child job did not run', {}, '2014-04-29 13:15:37',330                '2014-04-29 13:15:38')331        self_aborted_test = self._build_view(332                16, 'dummy_Abort', '104-user/host/dummy_Abort', 'ABORT',333                self_aborted_job_id,334                'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',335                'child job aborted', {'aborted_by': 'test_user'},336                '2014-04-29 13:15:39', '2014-04-29 13:15:40',337                job_started_time='2014-04-29 13:15:39',338                job_finished_time='2014-04-29 13:25:40')339        aborted_by_suite = self._build_view(340                17, 'dummy_AbortBySuite', '105-user/host/dummy_AbortBySuite',341                'RUNNING', aborted_by_suite_job_id,342                'lumpy-release/R27-3888.0.0/dummy/dummy_Abort',343                'aborted by suite', {'aborted_by': 'test_user'},344                '2014-04-29 13:15:39', '2014-04-29 13:15:40',345                job_started_time='2014-04-29 13:15:39',346                job_finished_time='2014-04-29 13:15:40')347        good_retry = self._build_view(348                18, 'dummy_RetryPass', '106-user/host/dummy_RetryPass', 'GOOD',349                good_retry_job_id,350                'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',351                '', {'retry_original_job_id': invalid_job_id_1},352                '2014-04-29 13:15:37',353                '2014-04-29 13:15:38', invalidates_test_idx=1)354        bad_retry = self._build_view(355                19, 'dummy_RetryFail', '107-user/host/dummy_RetryFail', 'FAIL',356                bad_retry_job_id,357                'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',358                'retry failed', {'retry_original_job_id': invalid_job_id_2},359                '2014-04-29 13:15:39', '2014-04-29 13:15:40',360                invalidates_test_idx=2)361        invalid_test_1 = self._build_view(362                1, 'dummy_RetryPass', '90-user/host/dummy_RetryPass', 'GOOD',363                invalid_job_id_1,364                'lumpy-release/R27-3888.0.0/dummy/dummy_RetryPass',365                'original test failed', {}, '2014-04-29 13:10:00',366                '2014-04-29 13:10:01')367        invalid_test_2 = self._build_view(368                2, 'dummy_RetryFail', '91-user/host/dummy_RetryFail', 'FAIL',369                invalid_job_id_2,370                'lumpy-release/R27-3888.0.0/dummy/dummy_RetryFail',371                'original test failed', {},372                '2014-04-29 13:10:03', '2014-04-29 13:10:04')373        test_views = [server_job_view, good_test]374        child_jobs = set([good_job_id])375        if include_bad_test:376            test_views.append(bad_test)377            child_jobs.add(bad_job_id)378        if include_warn_test:379            test_views.append(warn_test)380            child_jobs.add(warn_job_id)381        if include_timeout_test:382            test_views.append(timeout_test)383        if include_self_aborted_test:384            test_views.append(self_aborted_test)385            child_jobs.add(self_aborted_job_id)386        if include_good_retry:387            test_views.extend([good_retry, invalid_test_1])388            child_jobs.add(good_retry_job_id)389        if include_bad_retry:390            test_views.extend([bad_retry, invalid_test_2])391            child_jobs.add(bad_retry_job_id)392        if include_aborted_by_suite_test:393            test_views.append(aborted_by_suite)394            child_jobs.add(aborted_by_suite_job_id)395        self._mock_tko_get_detailed_test_views(test_views)396        self._mock_afe_get_jobs(suite_job_id, child_jobs)397        collector = run_suite.ResultCollector(398               'fake_server', self.afe, self.tko,399               'lumpy-release/R36-5788.0.0', 'lumpy', 'dummy', suite_job_id,400               return_code_function=run_suite._ReturnCodeComputer())401        collector.run()402        return collector403    def testEndToEndSuitePass(self):404        """Test it returns code OK when all test pass."""405        collector = self._end_to_end_test_helper()406        self.assertEqual(collector.return_code, run_suite.RETURN_CODES.OK)407    def testEndToEndSuiteWarn(self):408        """Test it returns code WARNING when there is a test that warns."""409        collector = self._end_to_end_test_helper(include_warn_test=True)410        self.assertEqual(collector.return_code, run_suite.RETURN_CODES.WARNING)411    def testEndToEndSuiteFail(self):412        """Test it returns code ERROR when there is a test that fails."""413        collector = self._end_to_end_test_helper(include_bad_test=True)414        self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR)415    def testEndToEndSuiteJobFail(self):416        """Test it returns code SUITE_FAILURE when only the suite job failed."""417        collector = self._end_to_end_test_helper(suite_job_status='ABORT')418        self.assertEqual(419                collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE)420        collector = self._end_to_end_test_helper(suite_job_status='ERROR')421        self.assertEqual(422                collector.return_code, run_suite.RETURN_CODES.INFRA_FAILURE)423    def testEndToEndRetry(self):424        """Test it returns correct code when a test was retried."""425        collector = self._end_to_end_test_helper(include_good_retry=True)426        self.assertEqual(427                collector.return_code, run_suite.RETURN_CODES.WARNING)428        collector = self._end_to_end_test_helper(include_good_retry=True,429                include_self_aborted_test=True)430        self.assertEqual(431                collector.return_code, run_suite.RETURN_CODES.ERROR)432        collector = self._end_to_end_test_helper(include_good_retry=True,433                include_bad_test=True)434        self.assertEqual(435                collector.return_code, run_suite.RETURN_CODES.ERROR)436        collector = self._end_to_end_test_helper(include_bad_retry=True)437        self.assertEqual(438                collector.return_code, run_suite.RETURN_CODES.ERROR)439    def testEndToEndSuiteTimeout(self):440        """Test it returns correct code when a child job timed out."""441        # a child job timed out before started, none failed.442        collector = self._end_to_end_test_helper(include_timeout_test=True)443        self.assertEqual(444                collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)445        # a child job timed out before started, and one test failed.446        collector = self._end_to_end_test_helper(447                include_bad_test=True, include_timeout_test=True)448        self.assertEqual(collector.return_code, run_suite.RETURN_CODES.ERROR)449        # a child job timed out before started, and one test warned.450        collector = self._end_to_end_test_helper(451                include_warn_test=True, include_timeout_test=True)452        self.assertEqual(collector.return_code,453                         run_suite.RETURN_CODES.SUITE_TIMEOUT)454        # a child job timed out before started, and one test was retried.455        collector = self._end_to_end_test_helper(include_good_retry=True,456                include_timeout_test=True)457        self.assertEqual(458                collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)459        # a child jot was aborted because suite timed out.460        collector = self._end_to_end_test_helper(461                include_aborted_by_suite_test=True)462        self.assertEqual(463                collector.return_code, run_suite.RETURN_CODES.OK)464        # suite job timed out.465        collector = self._end_to_end_test_helper(suite_job_timed_out=True)466        self.assertEqual(467                collector.return_code, run_suite.RETURN_CODES.SUITE_TIMEOUT)468class LogLinkUnittests(unittest.TestCase):469    """Test the LogLink"""470    def testGenerateBuildbotLinks(self):471        """Test LogLink GenerateBuildbotLinks"""472        log_link_a = run_suite.LogLink('mock_anchor', 'mock_server',473                                      'mock_job_string',474                                      bug_info=('mock_bug_id', 1),475                                      reason='mock_reason',476                                      retry_count=1,477                                      testname='mock_testname')478        # Generate a bug link and a log link when bug_info is present479        self.assertTrue(len(list(log_link_a.GenerateBuildbotLinks())) == 2)480        log_link_b = run_suite.LogLink('mock_anchor', 'mock_server',481                                      'mock_job_string_b',482                                      reason='mock_reason',483                                      retry_count=1,484                                      testname='mock_testname')485        # Generate a log link when there is no bug_info486        self.assertTrue(len(list(log_link_b.GenerateBuildbotLinks())) == 1)487class SimpleTimerUnittests(unittest.TestCase):488    """Test the simple timer."""489    def testPoll(self):490        """Test polling the timer."""491        interval_hours = 0.0001492        t = diagnosis_utils.SimpleTimer(interval_hours=interval_hours)493        deadline = t.deadline494        self.assertTrue(deadline is not None and495                        t.interval_hours == interval_hours)496        min_deadline = (datetime.now() +497                        datetime_base.timedelta(hours=interval_hours))498        time.sleep(interval_hours * 3600)499        self.assertTrue(t.poll())500        self.assertTrue(t.deadline >= min_deadline)501    def testBadInterval(self):502        """Test a bad interval."""503        t = diagnosis_utils.SimpleTimer(interval_hours=-1)504        self.assertTrue(t.deadline is None and t.poll() == False)505        t._reset()506        self.assertTrue(t.deadline is None and t.poll() == False)507class ArgumentParserUnittests(unittest.TestCase):508    """Tests for argument parser."""509    @unittest.expectedFailure510    def test_crbug_658013(self):511        """crbug.com/658013512        Expected failure due to http://bugs.python.org/issue9334513        """514        parser = run_suite.make_parser()515        args = [516            '--board', 'heli',517            '--build', 'trybot-heli-paladin/R56-8918.0.0-b1601',518            '--suite_name', 'test_that_wrapper',519            '--pool', 'suites',520            '--max_runtime_mins', '20',521            '--suite_args', '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline',522        ]523        def error_handler(msg):  # pylint: disable=missing-docstring524            self.fail('Argument parsing failed: ' + msg)525        parser.error = error_handler526        got = parser.parse_args(args)527        self.assertEqual(528            got.board, 'heli')529        self.assertEqual(530            got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601')531        self.assertEqual(532            got.suite_args,533            '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline')534    def test_crbug_658013b(self):535        """crbug.com/658013536        Unambiguous behavior.537        """538        parser = run_suite.make_parser()539        args = [540            '--board=heli',541            '--build=trybot-heli-paladin/R56-8918.0.0-b1601',542            '--suite_name=test_that_wrapper',543            '--pool=suites',544            '--max_runtime_mins=20',545            '--suite_args=-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline',546        ]547        def error_handler(msg):  # pylint: disable=missing-docstring548            self.fail('Argument parsing failed: ' + msg)549        parser.error = error_handler550        got = parser.parse_args(args)551        self.assertEqual(552            got.board, 'heli')553        self.assertEqual(554            got.build, 'trybot-heli-paladin/R56-8918.0.0-b1601')555        self.assertEqual(556            got.suite_args,557            '-b heli -i trybot-heli-paladin/R56-8918.0.0-b1601 :lab: suite:bvt-inline')558if __name__ == '__main__':...

test_event_order.py

Source:test_event_order.py

...41        self.event_order.append(test_skipped)42    def on_test_started(self, **kwargs):43        self.event_order.append(test_started)44    def test_happy_path(self):45        def run_suite():46            class Suite(TestSuite):47                def test_nothing(self):48                    pass49        run_suite()50        expect(self.event_order).to(equal([suite_started, test_started, test_ended, suite_ended]))51    def test_suite_setup_error(self):52        def run_suite():53            class Suite(TestSuite):54                def setup_suite(self):55                    raise RuntimeError("intentional")56                def test_nothing(self):57                    pass58        run_suite()59        expect(self.event_order).to(equal([suite_started, suite_erred, test_skipped, suite_ended]))60    def test_test_setup_error(self):61        def run_suite():62            class Suite(TestSuite):63                def setup(self):64                    raise RuntimeError("intentional")65                def test_should_not_run(self):66                    pass67        run_suite()68        expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))69    def test_test_error(self):70        def run_suite():71            class Suite(TestSuite):72                def test_go_boom(self):73                    raise RuntimeError("intentional")74        run_suite()75        expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))76    def test_test_failure(self):77        def run_suite():78            class Suite(TestSuite):79                def test_fails(self):80                    raise AssertionError("intentional")81        run_suite()82        expect(self.event_order).to(equal([suite_started, test_started, test_failed, test_ended, suite_ended]))83    def test_test_skip(self):84        def run_suite():85            class Suite(TestSuite):86                def test_fails(self):87                    raise TestSkipped("intentional")88        run_suite()89        expect(self.event_order).to(equal([suite_started, test_started, test_skipped, test_ended, suite_ended]))90    def test_test_teardown_failure(self):91        def run_suite():92            class Suite(TestSuite):93                def test_nothing(self):94                    pass95                def teardown(self):96                    raise RuntimeError("intentional")97        run_suite()98        expect(self.event_order).to(equal([suite_started, test_started, test_erred, test_ended, suite_ended]))99    def test_suite_teardown_failure(self):100        def run_suite():101            class Suite(TestSuite):102                def test_nothing(self):103                    pass104                def teardown_suite(self):105                    raise RuntimeError("intentional")106        run_suite()107        expect(self.event_order).to(equal([suite_started, test_started, test_ended, suite_erred, suite_ended]))108if "__main__" == __name__:...

runners.py

Source:runners.py

...39        except Exception, e:40            errors.append(f[0])41            traceback.print_exc()42    return passed, failed, errors43def run_suite(modules, smoke=False, report=False):44    passed    = []45    failed    = []46    errors    = []47    target    = 048    try:49        git_dir  = os.path.dirname(os.path.dirname(__file__))50        git_sha1 = ave.git.rev_list(git_dir, 1)[0]51    except Exception, e:52        git_sha1 = ''53    for m in modules:54        args = tuple()55        if type(m) == tuple:56            args = m[1]57            m = m[0]58        package = 'ave.common.%s' % m.__name__.split('.')[-1]59        functions = inspect.getmembers(m, inspect.isfunction)60        functions = [f for f in functions if re.match('t\d+', f[0])]61        if smoke:62            functions = [f for f in functions if hasattr(f[1], 'smoke')]63        target += len(functions)64        p,f,e = run_tests(functions, package, *args)65        passed.extend(p)66        failed.extend(f)67        errors.extend(e)68    if len(passed) != target:69        return vcsjob.FAILURES70    return vcsjob.OK71def all_smoke(report, so_path):72    modules = [73        tests.config, tests.cmd, tests.documentation, tests.spool,74        tests.connection, tests.control, (tests.fdtx, (so_path,))75    ]76    return run_suite(modules, True, report)77def all_config(report=False):78    return run_suite([tests.config], False, report)79def all_cmd(report=False):80    return run_suite([tests.cmd], False, report)81def all_documentation(report=False):82    return run_suite([tests.documentation], False, report)83def all_spool(report=False):84    # tests for irrelevant functionality. don't run85    return run_suite([tests.spool], False, report)86def all_connection(report=False):87    return run_suite([tests.connection], False, report)88def all_control(report=False):89    run_suite([tests.control_sync],  False, report)90    run_suite([tests.control_async], False, report)91#    for i in range(100000):92#        tests.control_async.t28()93#        print 'num fds: %d' % len(os.listdir('/proc/%d/fd' % os.getpid()))94#        gc.collect()95#        pass96def all_fdtx(so_path=None, report=False):97    return run_suite([(tests.fdtx, (so_path,))], False, report)98def all_exception(report=False):99    return run_suite([tests.exception], False, report)100def all_hickup(report=False):101    return run_suite([tests.hickup], False, report)102def all_stress():103    return tests.stress.t1()104def all_panotti(report=False):105    return run_suite([tests.panotti], False, report)106def all_process(report=False):107    return run_suite([tests.process], False, report)108def all_pipe(report=False):109    return run_suite([tests.pipe], False, report)110def all_daemon(report=False):111    return run_suite([tests.daemon], False, report)112def all_defjoin(report=False):113    return run_suite([tests.defjoin], False, report)114def all_async_rpc(report=False):...

test_gpu_updaters.py

Source:test_gpu_updaters.py

...16    def test_gpu_exact(self):17        variable_param = {'max_depth': [2, 6, 15], }18        for param in parameter_combinations(variable_param):19            param['tree_method'] = 'gpu_exact'20            gpu_results = run_suite(param, select_datasets=datasets)21            assert_results_non_increasing(gpu_results, 1e-2)22            param['tree_method'] = 'exact'23            cpu_results = run_suite(param, select_datasets=datasets)24            assert_gpu_results(cpu_results, gpu_results)25    def test_gpu_hist(self):26        test_param = parameter_combinations({'n_gpus': [1], 'max_depth': [2, 8],27                                             'max_leaves': [255, 4],28                                             'max_bin': [2, 256],29                                             'grow_policy': ['lossguide']})30        test_param.append({'single_precision_histogram': True})31        test_param.append({'min_child_weight': 0,32                           'lambda': 0})33        for param in test_param:34            param['tree_method'] = 'gpu_hist'35            gpu_results = run_suite(param, select_datasets=datasets)36            assert_results_non_increasing(gpu_results, 1e-2)37            param['tree_method'] = 'hist'38            cpu_results = run_suite(param, select_datasets=datasets)39            assert_gpu_results(cpu_results, gpu_results)40    @pytest.mark.mgpu41    def test_gpu_hist_mgpu(self):42        variable_param = {'n_gpus': [-1], 'max_depth': [2, 10],43                          'max_leaves': [255, 4],44                          'max_bin': [2, 256],45                          'grow_policy': ['lossguide'], 'debug_synchronize': [True]}46        for param in parameter_combinations(variable_param):47            param['tree_method'] = 'gpu_hist'48            gpu_results = run_suite(param, select_datasets=datasets)49            assert_results_non_increasing(gpu_results, 1e-2)50    @pytest.mark.mgpu51    def test_specified_gpu_id_gpu_update(self):52        variable_param = {'n_gpus': [1],53                          'gpu_id': [1],54                          'max_depth': [8],55                          'max_leaves': [255, 4],56                          'max_bin': [2, 64],57                          'grow_policy': ['lossguide'],58                          'tree_method': ['gpu_hist', 'gpu_exact']}59        for param in parameter_combinations(variable_param):60            gpu_results = run_suite(param, select_datasets=datasets)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.