Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use _replace method in autotest

Best Python code snippet using autotest_python

test_update_test_lifecycle.py

Source:test_update_test_lifecycle.py

...26        """27        Tests the validation of the 'test_fail_rates.acceptable' attribute.28        """29        with self.assertRaises(TypeError):30            config = self.CONFIG._replace(31                test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable="not a number"))32            update_test_lifecycle.validate_config(config)33        with self.assertRaises(ValueError):34            config = self.CONFIG._replace(35                test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=-1))36            update_test_lifecycle.validate_config(config)37        with self.assertRaises(ValueError):38            config = self.CONFIG._replace(39                test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=2))40            update_test_lifecycle.validate_config(config)41    def test_unacceptable_test_fail_rate(self):42        """43        Tests the validation of the 'test_fail_rates.unacceptable' attribute.44        """45        with self.assertRaises(TypeError):46            config = self.CONFIG._replace(47                test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable="not a number"))48            update_test_lifecycle.validate_config(config)49        with self.assertRaises(ValueError):50            config = self.CONFIG._replace(51                test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=-1))52            update_test_lifecycle.validate_config(config)53        with self.assertRaises(ValueError):54            config = self.CONFIG._replace(55                test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=2))56            update_test_lifecycle.validate_config(config)57    def test_test_fail_rates(self):58        """59        Tests the validation of the 'test_fail_rates' attribute.60        """61        with self.assertRaises(ValueError):62            config = self.CONFIG._replace(63                test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9,64                                                                     unacceptable=0.1))65            update_test_lifecycle.validate_config(config)66    def test_acceptable_task_fail_rate(self):67        """68        Tests the validation of the 'test_fail_rates.acceptable' attribute.69        """70        with self.assertRaises(TypeError):71            config = self.CONFIG._replace(72                task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable="not a number"))73            update_test_lifecycle.validate_config(config)74        with self.assertRaises(ValueError):75            config = self.CONFIG._replace(76                task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=-1))77            update_test_lifecycle.validate_config(config)78        with self.assertRaises(ValueError):79            config = self.CONFIG._replace(80                task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=2))81            update_test_lifecycle.validate_config(config)82    def test_unacceptable_task_fail_rate(self):83        """84        Tests the validation of the 'task_fail_rates.unacceptable' attribute.85        """86        with self.assertRaises(TypeError):87            config = self.CONFIG._replace(88                task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable="not a number"))89            update_test_lifecycle.validate_config(config)90        with self.assertRaises(ValueError):91            config = self.CONFIG._replace(92                task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=-1))93            update_test_lifecycle.validate_config(config)94        with self.assertRaises(ValueError):95            config = self.CONFIG._replace(96                task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=2))97            update_test_lifecycle.validate_config(config)98    def test_task_fail_rates(self):99        """100        Tests the validation of the 'task_fail_rates' attribute.101        """102        with self.assertRaises(ValueError):103            config = self.CONFIG._replace(104                task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9,105                                                                     unacceptable=0.1))106            update_test_lifecycle.validate_config(config)107    def test_acceptable_variant_fail_rate(self):108        """109        Tests the validation of the 'variant_fail_rates.acceptable' attribute.110        """111        with self.assertRaises(TypeError):112            config = self.CONFIG._replace(113                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(114                    acceptable="not a number"))115            update_test_lifecycle.validate_config(config)116        with self.assertRaises(ValueError):117            config = self.CONFIG._replace(118                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=-1))119            update_test_lifecycle.validate_config(config)120        with self.assertRaises(ValueError):121            config = self.CONFIG._replace(122                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=2))123            update_test_lifecycle.validate_config(config)124    def test_unacceptable_variant_fail_rate(self):125        """126        Tests the validation of the 'variant_fail_rates.unacceptable' attribute.127        """128        with self.assertRaises(TypeError):129            config = self.CONFIG._replace(130                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(131                    unacceptable="not a number"))132            update_test_lifecycle.validate_config(config)133        with self.assertRaises(ValueError):134            config = self.CONFIG._replace(135                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=-1))136            update_test_lifecycle.validate_config(config)137        with self.assertRaises(ValueError):138            config = self.CONFIG._replace(139                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=2))140            update_test_lifecycle.validate_config(config)141    def test_variant_fail_rates(self):142        """143        Tests the validation of the 'variant_fail_rates' attribute.144        """145        with self.assertRaises(ValueError):146            config = self.CONFIG._replace(147                variant_fail_rates=self.CONFIG.variant_fail_rates._replace(148                    acceptable=0.9, unacceptable=0.1))149            update_test_lifecycle.validate_config(config)150    def test_acceptable_distro_fail_rate(self):151        """152        Tests the validation of the 'distro_fail_rates.acceptable' attribute.153        """154        with self.assertRaises(TypeError):155            config = self.CONFIG._replace(156                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number"))157            update_test_lifecycle.validate_config(config)158        with self.assertRaises(ValueError):159            config = self.CONFIG._replace(160                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1))161            update_test_lifecycle.validate_config(config)162        with self.assertRaises(ValueError):163            config = self.CONFIG._replace(164                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2))165            update_test_lifecycle.validate_config(config)166    def test_unacceptable_distro_fail_rate(self):167        """168        Tests the validation of the 'distro_fail_rates.unacceptable' attribute.169        """170        with self.assertRaises(TypeError):171            config = self.CONFIG._replace(172                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(173                    unacceptable="not a number"))174            update_test_lifecycle.validate_config(config)175        with self.assertRaises(ValueError):176            config = self.CONFIG._replace(177                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=-1))178            update_test_lifecycle.validate_config(config)179        with self.assertRaises(ValueError):180            config = self.CONFIG._replace(181                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=2))182            update_test_lifecycle.validate_config(config)183    def test_distro_fail_rates(self):184        """185        Tests the validation of the 'distro_fail_rates' attribute.186        """187        with self.assertRaises(ValueError):188            config = self.CONFIG._replace(189                distro_fail_rates=self.CONFIG.distro_fail_rates._replace(190                    acceptable=0.9, unacceptable=0.1))191            update_test_lifecycle.validate_config(config)192    def test_reliable_min_runs(self):193        """194        Tests the validation of the 'reliable_min_runs' attribute.195        """196        with self.assertRaises(TypeError):197            config = self.CONFIG._replace(reliable_min_runs="not a number")198            update_test_lifecycle.validate_config(config)199        with self.assertRaises(ValueError):200            config = self.CONFIG._replace(reliable_min_runs=-1)201            update_test_lifecycle.validate_config(config)202        with self.assertRaises(ValueError):203            config = self.CONFIG._replace(reliable_min_runs=0)204            update_test_lifecycle.validate_config(config)205        with self.assertRaises(ValueError):206            config = self.CONFIG._replace(reliable_min_runs=1.5)207            update_test_lifecycle.validate_config(config)208    def test_reliable_time_period(self):209        """210        Tests the validation of the 'reliable_time_period' attribute.211        """212        with self.assertRaises(TypeError):213            config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta")214            update_test_lifecycle.validate_config(config)215        with self.assertRaises(ValueError):216            config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1))217            update_test_lifecycle.validate_config(config)218        with self.assertRaises(ValueError):219            config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0))220            update_test_lifecycle.validate_config(config)221        with self.assertRaises(ValueError):222            config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1))223            update_test_lifecycle.validate_config(config)224    def test_unreliable_min_runs(self):225        """226        Tests the validation of the 'unreliable_min_runs' attribute.227        """228        with self.assertRaises(TypeError):229            config = self.CONFIG._replace(unreliable_min_runs="not a number")230            update_test_lifecycle.validate_config(config)231        with self.assertRaises(ValueError):232            config = self.CONFIG._replace(unreliable_min_runs=-1)233            update_test_lifecycle.validate_config(config)234        with self.assertRaises(ValueError):235            config = self.CONFIG._replace(unreliable_min_runs=0)236            update_test_lifecycle.validate_config(config)237        with self.assertRaises(ValueError):238            config = self.CONFIG._replace(unreliable_min_runs=1.5)239            update_test_lifecycle.validate_config(config)240    def test_unreliable_time_period(self):241        """242        Tests the validation of the 'unreliable_time_period' attribute.243        """244        with self.assertRaises(TypeError):245            config = self.CONFIG._replace(unreliable_time_period="not a datetime.timedelta")246            update_test_lifecycle.validate_config(config)247        with self.assertRaises(ValueError):248            config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=-1))249            update_test_lifecycle.validate_config(config)250        with self.assertRaises(ValueError):251            config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=0))252            update_test_lifecycle.validate_config(config)253        with self.assertRaises(ValueError):254            config = self.CONFIG._replace(255                unreliable_time_period=datetime.timedelta(days=1, hours=1))256            update_test_lifecycle.validate_config(config)257class TestUpdateTags(unittest.TestCase):  # pylint: disable=too-many-public-methods258    """259    Tests for the update_tags() function.260    """261    CONFIG = update_test_lifecycle.Config(262        test_fail_rates=update_test_lifecycle.Rates(263            acceptable=0, unacceptable=1), task_fail_rates=update_test_lifecycle.Rates(264                acceptable=0, unacceptable=1), variant_fail_rates=update_test_lifecycle.Rates(265                    acceptable=0, unacceptable=1), distro_fail_rates=update_test_lifecycle.Rates(266                        acceptable=0, unacceptable=1), reliable_min_runs=2,267        reliable_time_period=datetime.timedelta(days=1), unreliable_min_runs=2,268        unreliable_time_period=datetime.timedelta(days=1))269    ENTRY = test_failures.ReportEntry(test="jstests/core/all.js", task="jsCore_WT",270                                      variant="linux-64", distro="rhel62", start_date=datetime.date(271                                          2017, 6, 3), end_date=datetime.date(2017, 6, 3),272                                      num_pass=0, num_fail=0)273    def assert_has_only_js_tests(self, lifecycle):274        """275        Raises an AssertionError exception if 'lifecycle' is not of the following form:276            selector:277              js_test:278                ...279        """280        self.assertIn("selector", lifecycle.raw)281        self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))282        self.assertIn("js_test", lifecycle.raw["selector"])283        self.assertEqual(1, len(lifecycle.raw["selector"]), msg=str(lifecycle.raw))284        return lifecycle.raw["selector"]["js_test"]285    def transition_from_reliable_to_unreliable(self, config, expected_tags):286        """287        Tests that update_tags() tags a formerly reliable combination as being unreliable.288        """289        initial_tags = collections.OrderedDict()290        lifecycle = ci_tags.TagsConfig.from_dict(291            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))292        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)293        self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle))294        tests = ["jstests/core/all.js"]295        report = test_failures.Report([296            self.ENTRY._replace(num_pass=0, num_fail=1),297            self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),298            self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),299            self.ENTRY._replace(num_pass=1, num_fail=0),300            self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),301        ])302        update_test_lifecycle.validate_config(config)303        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)304        updated_tags = self.assert_has_only_js_tests(lifecycle)305        self.assertEqual(updated_tags, expected_tags)306    def test_transition_test_from_reliable_to_unreliable(self):307        """308        Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.309        """310        config = self.CONFIG._replace(311            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1))312        self.transition_from_reliable_to_unreliable(config,313                                                    collections.OrderedDict([314                                                        ("jstests/core/all.js", ["unreliable"]),315                                                    ]))316    def test_transition_task_from_reliable_to_unreliable(self):317        """318        Tests that update_tags() tags a formerly reliable (test, task) combination as being319        unreliable.320        """321        config = self.CONFIG._replace(322            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1))323        self.transition_from_reliable_to_unreliable(config,324                                                    collections.OrderedDict([325                                                        ("jstests/core/all.js",326                                                         ["unreliable|jsCore_WT"]),327                                                    ]))328    def test_transition_variant_from_reliable_to_unreliable(self):329        """330        Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being331        unreliable.332        """333        config = self.CONFIG._replace(334            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1))335        self.transition_from_reliable_to_unreliable(config,336                                                    collections.OrderedDict([337                                                        ("jstests/core/all.js",338                                                         ["unreliable|jsCore_WT|linux-64"]),339                                                    ]))340    def test_transition_distro_from_reliable_to_unreliable(self):341        """342        Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination343        as being unreliable.344        """345        config = self.CONFIG._replace(346            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))347        self.transition_from_reliable_to_unreliable(config,348                                                    collections.OrderedDict([349                                                        ("jstests/core/all.js",350                                                         ["unreliable|jsCore_WT|linux-64|rhel62"]),351                                                    ]))352    def test_transition_from_reliable_to_unreliable(self):353        """354        Tests that update_tags() tags multiple formerly reliable combination as being unreliable.355        """356        config = self.CONFIG._replace(357            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),358            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),359            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),360            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))361        self.transition_from_reliable_to_unreliable(config,362                                                    collections.OrderedDict([363                                                        ("jstests/core/all.js", [364                                                            "unreliable",365                                                            "unreliable|jsCore_WT",366                                                            "unreliable|jsCore_WT|linux-64",367                                                            "unreliable|jsCore_WT|linux-64|rhel62",368                                                        ]),369                                                    ]))370    def transition_from_unreliable_to_reliable(self, config, initial_tags):371        """372        Tests that update_tags() untags a formerly unreliable combination after it has become373        reliable again.374        """375        lifecycle = ci_tags.TagsConfig.from_dict(376            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))377        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)378        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))379        tests = ["jstests/core/all.js"]380        report = test_failures.Report([381            self.ENTRY._replace(num_pass=1, num_fail=0),382            self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),383            self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),384            self.ENTRY._replace(num_pass=0, num_fail=1),385            self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),386        ])387        update_test_lifecycle.validate_config(config)388        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)389        updated_tags = self.assert_has_only_js_tests(lifecycle)390        self.assertEqual(updated_tags, collections.OrderedDict())391    def test_non_running_in_reliable_period_is_reliable(self):392        """393        Tests that tests that have a failure rate above the unacceptable rate during the unreliable394        period but haven't run during the reliable period are marked as reliable.395        """396        # Unreliable period is 2 days: 2017-06-03 to 2017-06-04.397        # Reliable period is 1 day: 2016-06-04.398        reliable_period_date = datetime.date(2017, 6, 4)399        config = self.CONFIG._replace(400            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),401            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),402            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),403            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),404            unreliable_time_period=datetime.timedelta(days=2))405        tests = ["jstests/core/all.js"]406        initial_tags = collections.OrderedDict([407            ("jstests/core/all.js", [408                "unreliable",409                "unreliable|jsCore_WT",410                "unreliable|jsCore_WT|linux-64",411                "unreliable|jsCore_WT|linux-64|rhel62",412            ]),413        ])414        lifecycle = ci_tags.TagsConfig.from_dict(415            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))416        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)417        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))418        # The test did not run on the reliable period on linux-64.419        report = test_failures.Report([420            # Failing.421            self.ENTRY._replace(num_pass=0, num_fail=2),422            # Passing on a different variant.423            self.ENTRY._replace(start_date=reliable_period_date, end_date=reliable_period_date,424                                num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),425        ])426        update_test_lifecycle.validate_config(config)427        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)428        updated_tags = self.assert_has_only_js_tests(lifecycle)429        # The tags for variant and distro have been removed.430        self.assertEqual(updated_tags,431                         collections.OrderedDict([("jstests/core/all.js",432                                                   ["unreliable", "unreliable|jsCore_WT"])]))433    def test_non_running_at_all_is_reliable(self):434        """435        Tests that tests that are tagged as unreliable but no longer running (either during the436        reliable or the unreliable period) have their tags removed.437        """438        config = self.CONFIG439        tests = ["jstests/core/all.js", "jstests/core/all2.js"]440        initial_tags = collections.OrderedDict([441            ("jstests/core/all2.js", [442                "unreliable",443                "unreliable|jsCore_WT",444                "unreliable|jsCore_WT|linux-64",445                "unreliable|jsCore_WT|linux-64|rhel62",446            ]),447        ])448        lifecycle = ci_tags.TagsConfig.from_dict(449            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))450        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)451        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))452        # all2.js did not run at all453        report = test_failures.Report([self.ENTRY])454        update_test_lifecycle.validate_config(config)455        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)456        updated_tags = self.assert_has_only_js_tests(lifecycle)457        # The tags for variant and distro have been removed.458        self.assertEqual(updated_tags, collections.OrderedDict([]))459    def test_transition_test_from_unreliable_to_reliable(self):460        """461        Tests that update_tags() untags a formerly unreliable (test,) combination after it has462        become reliable again.463        """464        config = self.CONFIG._replace(465            test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9))466        self.transition_from_unreliable_to_reliable(config,467                                                    collections.OrderedDict([468                                                        ("jstests/core/all.js", ["unreliable"]),469                                                    ]))470    def test_transition_task_from_unreliable_to_reliable(self):471        """472        Tests that update_tags() untags a formerly unreliable (test, task) combination after it has473        become reliable again.474        """475        config = self.CONFIG._replace(476            task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9))477        self.transition_from_unreliable_to_reliable(config,478                                                    collections.OrderedDict([479                                                        ("jstests/core/all.js",480                                                         ["unreliable|jsCore_WT"]),481                                                    ]))482    def test_transition_variant_from_unreliable_to_reliable(self):483        """484        Tests that update_tags() untags a formerly unreliable (test, task, variant) combination485        after it has become reliable again.486        """487        config = self.CONFIG._replace(488            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9))489        self.transition_from_unreliable_to_reliable(config,490                                                    collections.OrderedDict([491                                                        ("jstests/core/all.js",492                                                         ["unreliable|jsCore_WT|linux-64"]),493                                                    ]))494    def test_transition_distro_from_unreliable_to_reliable(self):495        """496        Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)497        combination after it has become reliable again.498        """499        config = self.CONFIG._replace(500            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))501        self.transition_from_unreliable_to_reliable(config,502                                                    collections.OrderedDict([503                                                        ("jstests/core/all.js",504                                                         ["unreliable|jsCore_WT|linux-64|rhel62"]),505                                                    ]))506    def test_transition_from_unreliable_to_reliable(self):507        """508        Tests that update_tags() untags multiple formerly unreliable combination after it has become509        reliable again.510        """511        config = self.CONFIG._replace(512            test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),513            task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),514            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),515            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))516        self.transition_from_unreliable_to_reliable(config,517                                                    collections.OrderedDict([518                                                        ("jstests/core/all.js", [519                                                            "unreliable",520                                                            "unreliable|jsCore_WT",521                                                            "unreliable|jsCore_WT|linux-64",522                                                            "unreliable|jsCore_WT|linux-64|rhel62",523                                                        ]),524                                                    ]))525    def test_remain_reliable(self):526        """527        Tests that update_tags() preserves the absence of tags for reliable combinations.528        """529        config = self.CONFIG._replace(530            test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),531            task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),532            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),533            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))534        initial_tags = collections.OrderedDict()535        lifecycle = ci_tags.TagsConfig.from_dict(536            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))537        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)538        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))539        tests = ["jstests/core/all.js"]540        report = test_failures.Report([541            self.ENTRY._replace(num_pass=1, num_fail=0),542            self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),543            self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),544            self.ENTRY._replace(num_pass=0, num_fail=1),545            self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),546        ])547        update_test_lifecycle.validate_config(config)548        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)549        updated_tags = self.assert_has_only_js_tests(lifecycle)550        self.assertEqual(updated_tags, initial_tags)551    def test_remain_unreliable(self):552        """553        Tests that update_tags() preserves the tags for unreliable combinations.554        """555        config = self.CONFIG._replace(556            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),557            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),558            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),559            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))560        initial_tags = collections.OrderedDict([561            ("jstests/core/all.js", [562                "unreliable",563                "unreliable|jsCore_WT",564                "unreliable|jsCore_WT|linux-64",565                "unreliable|jsCore_WT|linux-64|rhel62",566            ]),567        ])568        lifecycle = ci_tags.TagsConfig.from_dict(569            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))570        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)571        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))572        tests = ["jstests/core/all.js"]573        report = test_failures.Report([574            self.ENTRY._replace(num_pass=0, num_fail=1),575            self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),576            self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),577            self.ENTRY._replace(num_pass=1, num_fail=0),578            self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),579        ])580        update_test_lifecycle.validate_config(config)581        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)582        updated_tags = self.assert_has_only_js_tests(lifecycle)583        self.assertEqual(updated_tags, initial_tags)584    def test_obeys_reliable_min_runs(self):585        """586        Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.587        """588        config = self.CONFIG._replace(589            test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),590            task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),591            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),592            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9),593            reliable_min_runs=100)594        self.transition_from_unreliable_to_reliable(config,595                                                    collections.OrderedDict([596                                                        ("jstests/core/all.js", [597                                                            "unreliable",598                                                            "unreliable|jsCore_WT",599                                                            "unreliable|jsCore_WT|linux-64",600                                                            "unreliable|jsCore_WT|linux-64|rhel62",601                                                        ]),602                                                    ]))603    def test_obeys_reliable_time_period(self):604        """605        Tests that update_tags() ignores passes from before 'reliable_time_period'.606        """607        config = self.CONFIG._replace(608            test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),609            task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),610            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),611            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))612        initial_tags = collections.OrderedDict()613        lifecycle = ci_tags.TagsConfig.from_dict(614            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))615        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)616        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))617        tests = ["jstests/core/all.js"]618        report = test_failures.Report([619            self.ENTRY._replace(620                start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)),621                end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=1,622                num_fail=0),623            self.ENTRY._replace(624                start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)),625                end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=1,626                num_fail=0),627            self.ENTRY._replace(num_pass=0, num_fail=1),628            self.ENTRY._replace(num_pass=0, num_fail=1),629            self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),630            self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),631            self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),632        ])633        update_test_lifecycle.validate_config(config)634        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)635        updated_tags = self.assert_has_only_js_tests(lifecycle)636        self.assertEqual(updated_tags,637                         collections.OrderedDict([638                             ("jstests/core/all.js", [639                                 "unreliable",640                                 "unreliable|jsCore_WT",641                                 "unreliable|jsCore_WT|linux-64",642                                 "unreliable|jsCore_WT|linux-64|rhel62",643                             ]),644                         ]))645    def test_obeys_unreliable_min_runs(self):646        """647        Tests that update_tags() only considers a test unreliable if it has more than648        'unreliable_min_runs'.649        """650        config = self.CONFIG._replace(651            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),652            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),653            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),654            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),655            unreliable_min_runs=100)656        initial_tags = collections.OrderedDict()657        lifecycle = ci_tags.TagsConfig.from_dict(658            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))659        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)660        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))661        tests = ["jstests/core/all.js"]662        report = test_failures.Report([663            self.ENTRY._replace(num_pass=0, num_fail=1),664            self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),665            self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),666            self.ENTRY._replace(num_pass=1, num_fail=0),667            self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),668        ])669        update_test_lifecycle.validate_config(config)670        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)671        updated_tags = self.assert_has_only_js_tests(lifecycle)672        self.assertEqual(updated_tags, initial_tags)673    def test_obeys_unreliable_time_period(self):674        """675        Tests that update_tags() ignores failures from before 'unreliable_time_period'.676        """677        config = self.CONFIG._replace(678            test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),679            task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),680            variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),681            distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))682        initial_tags = collections.OrderedDict([683            ("jstests/core/all.js", [684                "unreliable",685                "unreliable|jsCore_WT",686                "unreliable|jsCore_WT|linux-64",687                "unreliable|jsCore_WT|linux-64|rhel62",688            ]),689        ])690        lifecycle = ci_tags.TagsConfig.from_dict(691            dict(selector=dict(js_test=copy.deepcopy(initial_tags))))692        summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)693        self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))694        tests = ["jstests/core/all.js"]695        report = test_failures.Report([696            self.ENTRY._replace(697                start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)),698                end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=0,699                num_fail=1),700            self.ENTRY._replace(701                start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)),702                end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=0,703                num_fail=1),704            self.ENTRY._replace(num_pass=1, num_fail=0),705            self.ENTRY._replace(num_pass=1, num_fail=0),706            self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),707            self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),708            self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),709        ])710        update_test_lifecycle.validate_config(config)711        update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)712        updated_tags = self.assert_has_only_js_tests(lifecycle)713        self.assertEqual(updated_tags, collections.OrderedDict())714class TestCombinationHelpers(unittest.TestCase):715    def test_from_entry(self):716        entry = test_failures._ReportEntry("testA", "taskA", "variantA", "distroA",717                                           datetime.date.today(), datetime.date.today(), 0, 0)718        combination = update_test_lifecycle._test_combination_from_entry(719            entry, test_failures.Report.TEST)720        self.assertEqual(combination, ("testA", ))721        combination = update_test_lifecycle._test_combination_from_entry(722            entry, test_failures.Report.TEST_TASK)...

gen_build_yaml.py

Source:gen_build_yaml.py

...21    'fullstack includes_proxy dns_resolver name_resolution secure platforms ci_mac tracing exclude_configs exclude_iomgrs large_writes enables_compression supports_compression is_inproc is_http2 supports_proxy_auth supports_write_buffering client_channel')22default_unsecure_fixture_options = FixtureOptions(23    True, False, True, True, False, ['windows', 'linux', 'mac', 'posix'],24    True, False, [], [], True, False, True, False, True, False, True, True)25socketpair_unsecure_fixture_options = default_unsecure_fixture_options._replace(26    fullstack=False, dns_resolver=False, client_channel=False)27default_secure_fixture_options = default_unsecure_fixture_options._replace(28    secure=True)29uds_fixture_options = default_unsecure_fixture_options._replace(30    dns_resolver=False, platforms=['linux', 'mac', 'posix'],31    exclude_iomgrs=['uv'])32local_fixture_options = default_secure_fixture_options._replace(33    dns_resolver=False, platforms=['linux', 'mac', 'posix'],34    exclude_iomgrs=['uv'])35fd_unsecure_fixture_options = default_unsecure_fixture_options._replace(36    dns_resolver=False, fullstack=False, platforms=['linux', 'mac', 'posix'],37    exclude_iomgrs=['uv'], client_channel=False)38inproc_fixture_options = default_secure_fixture_options._replace(39    dns_resolver=False, fullstack=False, name_resolution=False,40    supports_compression=False, is_inproc=True, is_http2=False,41    supports_write_buffering=False, client_channel=False)42# maps fixture name to whether it requires the security library43END2END_FIXTURES = {44    'h2_compress': default_unsecure_fixture_options._replace(enables_compression=True),45    'h2_census': default_unsecure_fixture_options,46     # This cmake target is disabled for now because it depends on OpenCensus,47     # which is Bazel-only.48     # 'h2_load_reporting': default_unsecure_fixture_options,49    'h2_fakesec': default_secure_fixture_options._replace(ci_mac=False),50    'h2_fd': fd_unsecure_fixture_options,51    'h2_full': default_unsecure_fixture_options,52    'h2_full+pipe': default_unsecure_fixture_options._replace(53        platforms=['linux'], exclude_iomgrs=['uv']),54    'h2_full+trace': default_unsecure_fixture_options._replace(tracing=True),55    'h2_full+workarounds': default_unsecure_fixture_options,56    'h2_http_proxy': default_unsecure_fixture_options._replace(57        ci_mac=False, exclude_iomgrs=['uv'], supports_proxy_auth=True),58    'h2_oauth2': default_secure_fixture_options._replace(59        ci_mac=False, exclude_iomgrs=['uv']),60    'h2_proxy': default_unsecure_fixture_options._replace(61        includes_proxy=True, ci_mac=False, exclude_iomgrs=['uv']),62    'h2_sockpair_1byte': socketpair_unsecure_fixture_options._replace(63        ci_mac=False, exclude_configs=['msan'], large_writes=False,64        exclude_iomgrs=['uv']),65    'h2_sockpair': socketpair_unsecure_fixture_options._replace(66        ci_mac=False, exclude_iomgrs=['uv']),67    'h2_sockpair+trace': socketpair_unsecure_fixture_options._replace(68        ci_mac=False, tracing=True, large_writes=False, exclude_iomgrs=['uv']),69    'h2_ssl': default_secure_fixture_options,70    'h2_ssl_cred_reload': default_secure_fixture_options,71    'h2_spiffe': default_secure_fixture_options,72    'h2_local_uds': local_fixture_options,73    'h2_local_ipv4': local_fixture_options,74    'h2_local_ipv6': local_fixture_options,75    'h2_ssl_proxy': default_secure_fixture_options._replace(76        includes_proxy=True, ci_mac=False, exclude_iomgrs=['uv']),77    'h2_uds': uds_fixture_options,78    'inproc': inproc_fixture_options79}80TestOptions = collections.namedtuple(81    'TestOptions',82    'needs_fullstack needs_dns needs_names proxyable secure traceable cpu_cost exclude_iomgrs large_writes flaky allows_compression needs_compression exclude_inproc needs_http2 needs_proxy_auth needs_write_buffering needs_client_channel')83default_test_options = TestOptions(84    False, False, False, True, False, True, 1.0, [], False, False, True,85    False, False, False, False, False, False)86connectivity_test_options = default_test_options._replace(87    needs_fullstack=True)88LOWCPU = 0.189# maps test names to options90END2END_TESTS = {91    'authority_not_supported': default_test_options,92    'bad_hostname': default_test_options._replace(needs_names=True),93    'bad_ping': connectivity_test_options._replace(proxyable=False),94    'binary_metadata': default_test_options._replace(cpu_cost=LOWCPU),95    'resource_quota_server': default_test_options._replace(96        large_writes=True, proxyable=False, allows_compression=False),97    'call_creds': default_test_options._replace(secure=True),98    'cancel_after_accept': default_test_options._replace(cpu_cost=LOWCPU),99    'cancel_after_client_done': default_test_options._replace(cpu_cost=LOWCPU),100    'cancel_after_invoke': default_test_options._replace(cpu_cost=LOWCPU),101    'cancel_after_round_trip': default_test_options._replace(cpu_cost=LOWCPU),102    'cancel_before_invoke': default_test_options._replace(cpu_cost=LOWCPU),103    'cancel_in_a_vacuum': default_test_options._replace(cpu_cost=LOWCPU),104    'cancel_with_status': default_test_options._replace(cpu_cost=LOWCPU),105    'compressed_payload': default_test_options._replace(proxyable=False,106                                                        needs_compression=True),107    'connectivity': connectivity_test_options._replace(needs_names=True,108        proxyable=False, cpu_cost=LOWCPU, exclude_iomgrs=['uv']),109    'channelz': default_test_options,110    'default_host': default_test_options._replace(111        needs_fullstack=True, needs_dns=True, needs_names=True),112    'call_host_override': default_test_options._replace(113        needs_fullstack=True, needs_dns=True, needs_names=True),114    'disappearing_server': connectivity_test_options._replace(flaky=True,115                                                              needs_names=True),116    'empty_batch': default_test_options._replace(cpu_cost=LOWCPU),117    'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU),118    'filter_call_init_fails': default_test_options,119    'filter_context': default_test_options,120    'filter_latency': default_test_options._replace(cpu_cost=LOWCPU),121    'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU),122    'graceful_server_shutdown': default_test_options._replace(123        cpu_cost=LOWCPU, exclude_inproc=True),124    'hpack_size': default_test_options._replace(proxyable=False,125                                                traceable=False,126                                                cpu_cost=LOWCPU),127    'high_initial_seqno': default_test_options._replace(cpu_cost=LOWCPU),128    'idempotent_request': default_test_options,129    'invoke_large_request': default_test_options,130    'keepalive_timeout': default_test_options._replace(proxyable=False,131                                                       cpu_cost=LOWCPU,132                                                       needs_http2=True),133    'large_metadata': default_test_options,134    'max_concurrent_streams': default_test_options._replace(135        proxyable=False, cpu_cost=LOWCPU, exclude_inproc=True),136    'max_connection_age': default_test_options._replace(cpu_cost=LOWCPU,137                                                        exclude_inproc=True),138    'max_connection_idle': connectivity_test_options._replace(139        proxyable=False, exclude_iomgrs=['uv'], cpu_cost=LOWCPU),140    'max_message_length': default_test_options._replace(cpu_cost=LOWCPU),141    'negative_deadline': default_test_options,142    'no_error_on_hotpath': default_test_options._replace(proxyable=False),143    'no_logging': default_test_options._replace(traceable=False),144    'no_op': default_test_options,145    'payload': default_test_options,146    # This cmake target is disabled for now because it depends on OpenCensus,147    # which is Bazel-only.148    # 'load_reporting_hook': default_test_options,149    'ping_pong_streaming': default_test_options._replace(cpu_cost=LOWCPU),150    'ping': connectivity_test_options._replace(proxyable=False,151                                               cpu_cost=LOWCPU),152    'proxy_auth': default_test_options._replace(needs_proxy_auth=True),153    'registered_call': default_test_options,154    'request_with_flags': default_test_options._replace(155        proxyable=False, cpu_cost=LOWCPU),156    'request_with_payload': default_test_options._replace(cpu_cost=LOWCPU),157    # TODO(roth): Remove proxyable=False for all retry tests once we158    # have a way for the proxy to propagate the fact that trailing159    # metadata is available when initial metadata is returned.160    # See https://github.com/grpc/grpc/issues/14467 for context.161    'retry': default_test_options._replace(cpu_cost=LOWCPU,162                                           needs_client_channel=True,163                                           proxyable=False),164    'retry_cancellation': default_test_options._replace(165        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),166    'retry_disabled': default_test_options._replace(cpu_cost=LOWCPU,167                                                    needs_client_channel=True,168                                                    proxyable=False),169    'retry_exceeds_buffer_size_in_initial_batch': default_test_options._replace(170        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),171    'retry_exceeds_buffer_size_in_subsequent_batch':172        default_test_options._replace(cpu_cost=LOWCPU,173                                      needs_client_channel=True,174                                      proxyable=False),175    'retry_non_retriable_status': default_test_options._replace(176        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),177    'retry_non_retriable_status_before_recv_trailing_metadata_started':178        default_test_options._replace(179            cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),180    'retry_recv_initial_metadata': default_test_options._replace(181        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),182    'retry_recv_message': default_test_options._replace(183        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),184    'retry_server_pushback_delay': default_test_options._replace(185        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),186    'retry_server_pushback_disabled': default_test_options._replace(187        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),188    'retry_streaming': default_test_options._replace(cpu_cost=LOWCPU,189                                                     needs_client_channel=True,190                                                     proxyable=False),191    'retry_streaming_after_commit': default_test_options._replace(192        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),193    'retry_streaming_succeeds_before_replay_finished':194        default_test_options._replace(cpu_cost=LOWCPU,195                                      needs_client_channel=True,196                                      proxyable=False),197    'retry_throttled': default_test_options._replace(cpu_cost=LOWCPU,198                                                     needs_client_channel=True,199                                                     proxyable=False),200    'retry_too_many_attempts': default_test_options._replace(201        cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),202    'server_finishes_request': default_test_options._replace(cpu_cost=LOWCPU),203    'shutdown_finishes_calls': default_test_options._replace(cpu_cost=LOWCPU),204    'shutdown_finishes_tags': default_test_options._replace(cpu_cost=LOWCPU),205    'simple_cacheable_request': default_test_options._replace(cpu_cost=LOWCPU),206    'stream_compression_compressed_payload': default_test_options._replace(207        proxyable=False, exclude_inproc=True),208    'stream_compression_payload': default_test_options._replace(209        exclude_inproc=True),210    'stream_compression_ping_pong_streaming': default_test_options._replace(211        exclude_inproc=True),212    'simple_delayed_request': connectivity_test_options,213    'simple_metadata': default_test_options,214    'simple_request': default_test_options,215    'streaming_error_response': default_test_options._replace(cpu_cost=LOWCPU),216    'trailing_metadata': default_test_options,217    'workaround_cronet_compression': default_test_options,218    'write_buffering': default_test_options._replace(219        cpu_cost=LOWCPU, needs_write_buffering=True),220    'write_buffering_at_end': default_test_options._replace(221        cpu_cost=LOWCPU, needs_write_buffering=True),222}223def compatible(f, t):224  if END2END_TESTS[t].needs_fullstack:225    if not END2END_FIXTURES[f].fullstack:226      return False227  if END2END_TESTS[t].needs_dns:228    if not END2END_FIXTURES[f].dns_resolver:229      return False230  if END2END_TESTS[t].needs_names:231    if not END2END_FIXTURES[f].name_resolution:232      return False233  if not END2END_TESTS[t].proxyable:234    if END2END_FIXTURES[f].includes_proxy:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.