How to use _replace method in localstack

Best Python code snippet using localstack_python

test_update_test_lifecycle.py

Source:test_update_test_lifecycle.py Github

copy

Full Screen

...26 """27 Tests the validation of the 'test_fail_rates.acceptable' attribute.28 """29 with self.assertRaises(TypeError):30 config = self.CONFIG._replace(31 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable="not a number"))32 update_test_lifecycle.validate_config(config)33 with self.assertRaises(ValueError):34 config = self.CONFIG._replace(35 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=-1))36 update_test_lifecycle.validate_config(config)37 with self.assertRaises(ValueError):38 config = self.CONFIG._replace(39 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=2))40 update_test_lifecycle.validate_config(config)41 def test_unacceptable_test_fail_rate(self):42 """43 Tests the validation of the 'test_fail_rates.unacceptable' attribute.44 """45 with self.assertRaises(TypeError):46 config = self.CONFIG._replace(47 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable="not a number"))48 update_test_lifecycle.validate_config(config)49 with self.assertRaises(ValueError):50 config = self.CONFIG._replace(51 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=-1))52 update_test_lifecycle.validate_config(config)53 with self.assertRaises(ValueError):54 config = self.CONFIG._replace(55 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=2))56 update_test_lifecycle.validate_config(config)57 def test_test_fail_rates(self):58 """59 Tests the validation of the 'test_fail_rates' attribute.60 """61 with self.assertRaises(ValueError):62 config = self.CONFIG._replace(63 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9,64 unacceptable=0.1))65 update_test_lifecycle.validate_config(config)66 def test_acceptable_task_fail_rate(self):67 """68 Tests the validation of the 'test_fail_rates.acceptable' attribute.69 """70 with self.assertRaises(TypeError):71 config = self.CONFIG._replace(72 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable="not a number"))73 update_test_lifecycle.validate_config(config)74 with self.assertRaises(ValueError):75 config = self.CONFIG._replace(76 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=-1))77 update_test_lifecycle.validate_config(config)78 with self.assertRaises(ValueError):79 config = self.CONFIG._replace(80 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=2))81 update_test_lifecycle.validate_config(config)82 def test_unacceptable_task_fail_rate(self):83 """84 Tests the validation of the 'task_fail_rates.unacceptable' attribute.85 """86 with self.assertRaises(TypeError):87 config = self.CONFIG._replace(88 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable="not a number"))89 update_test_lifecycle.validate_config(config)90 with self.assertRaises(ValueError):91 config = self.CONFIG._replace(92 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=-1))93 update_test_lifecycle.validate_config(config)94 with self.assertRaises(ValueError):95 config = self.CONFIG._replace(96 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=2))97 update_test_lifecycle.validate_config(config)98 def test_task_fail_rates(self):99 """100 Tests the validation of the 'task_fail_rates' attribute.101 """102 with self.assertRaises(ValueError):103 config = self.CONFIG._replace(104 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9,105 unacceptable=0.1))106 update_test_lifecycle.validate_config(config)107 def test_acceptable_variant_fail_rate(self):108 """109 Tests the validation of the 'variant_fail_rates.acceptable' attribute.110 """111 with self.assertRaises(TypeError):112 config = self.CONFIG._replace(113 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(114 acceptable="not a number"))115 update_test_lifecycle.validate_config(config)116 with self.assertRaises(ValueError):117 config = self.CONFIG._replace(118 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=-1))119 update_test_lifecycle.validate_config(config)120 with self.assertRaises(ValueError):121 config = self.CONFIG._replace(122 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=2))123 update_test_lifecycle.validate_config(config)124 def test_unacceptable_variant_fail_rate(self):125 """126 Tests the validation of the 'variant_fail_rates.unacceptable' attribute.127 """128 with self.assertRaises(TypeError):129 config = self.CONFIG._replace(130 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(131 unacceptable="not a number"))132 update_test_lifecycle.validate_config(config)133 with self.assertRaises(ValueError):134 config = self.CONFIG._replace(135 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=-1))136 update_test_lifecycle.validate_config(config)137 with self.assertRaises(ValueError):138 config = self.CONFIG._replace(139 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=2))140 update_test_lifecycle.validate_config(config)141 def test_variant_fail_rates(self):142 """143 Tests the validation of the 'variant_fail_rates' attribute.144 """145 with self.assertRaises(ValueError):146 config = self.CONFIG._replace(147 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(148 acceptable=0.9, unacceptable=0.1))149 update_test_lifecycle.validate_config(config)150 def test_acceptable_distro_fail_rate(self):151 """152 Tests the validation of the 'distro_fail_rates.acceptable' attribute.153 """154 with self.assertRaises(TypeError):155 config = self.CONFIG._replace(156 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number"))157 update_test_lifecycle.validate_config(config)158 with self.assertRaises(ValueError):159 config = self.CONFIG._replace(160 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1))161 update_test_lifecycle.validate_config(config)162 with self.assertRaises(ValueError):163 config = self.CONFIG._replace(164 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2))165 update_test_lifecycle.validate_config(config)166 def test_unacceptable_distro_fail_rate(self):167 """168 Tests the validation of the 'distro_fail_rates.unacceptable' attribute.169 """170 with self.assertRaises(TypeError):171 config = self.CONFIG._replace(172 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(173 unacceptable="not a number"))174 update_test_lifecycle.validate_config(config)175 with self.assertRaises(ValueError):176 config = self.CONFIG._replace(177 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=-1))178 update_test_lifecycle.validate_config(config)179 with self.assertRaises(ValueError):180 config = self.CONFIG._replace(181 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=2))182 update_test_lifecycle.validate_config(config)183 def test_distro_fail_rates(self):184 """185 Tests the validation of the 'distro_fail_rates' attribute.186 """187 with self.assertRaises(ValueError):188 config = self.CONFIG._replace(189 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(190 acceptable=0.9, unacceptable=0.1))191 update_test_lifecycle.validate_config(config)192 def test_reliable_min_runs(self):193 """194 Tests the validation of the 'reliable_min_runs' attribute.195 """196 with self.assertRaises(TypeError):197 config = self.CONFIG._replace(reliable_min_runs="not a number")198 update_test_lifecycle.validate_config(config)199 with self.assertRaises(ValueError):200 config = self.CONFIG._replace(reliable_min_runs=-1)201 update_test_lifecycle.validate_config(config)202 with self.assertRaises(ValueError):203 config = self.CONFIG._replace(reliable_min_runs=0)204 update_test_lifecycle.validate_config(config)205 with self.assertRaises(ValueError):206 config = self.CONFIG._replace(reliable_min_runs=1.5)207 update_test_lifecycle.validate_config(config)208 def test_reliable_time_period(self):209 """210 Tests the validation of the 'reliable_time_period' attribute.211 """212 with self.assertRaises(TypeError):213 config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta")214 update_test_lifecycle.validate_config(config)215 with self.assertRaises(ValueError):216 config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1))217 update_test_lifecycle.validate_config(config)218 with self.assertRaises(ValueError):219 config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0))220 update_test_lifecycle.validate_config(config)221 with self.assertRaises(ValueError):222 config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1))223 update_test_lifecycle.validate_config(config)224 def test_unreliable_min_runs(self):225 """226 Tests the validation of the 'unreliable_min_runs' attribute.227 """228 with self.assertRaises(TypeError):229 config = self.CONFIG._replace(unreliable_min_runs="not a number")230 update_test_lifecycle.validate_config(config)231 with self.assertRaises(ValueError):232 config = self.CONFIG._replace(unreliable_min_runs=-1)233 update_test_lifecycle.validate_config(config)234 with self.assertRaises(ValueError):235 config = self.CONFIG._replace(unreliable_min_runs=0)236 update_test_lifecycle.validate_config(config)237 with self.assertRaises(ValueError):238 config = self.CONFIG._replace(unreliable_min_runs=1.5)239 update_test_lifecycle.validate_config(config)240 def test_unreliable_time_period(self):241 """242 Tests the validation of the 'unreliable_time_period' attribute.243 """244 with self.assertRaises(TypeError):245 config = self.CONFIG._replace(unreliable_time_period="not a datetime.timedelta")246 update_test_lifecycle.validate_config(config)247 with self.assertRaises(ValueError):248 config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=-1))249 update_test_lifecycle.validate_config(config)250 with self.assertRaises(ValueError):251 config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=0))252 update_test_lifecycle.validate_config(config)253 with self.assertRaises(ValueError):254 config = self.CONFIG._replace(255 unreliable_time_period=datetime.timedelta(days=1, hours=1))256 update_test_lifecycle.validate_config(config)257class TestUpdateTags(unittest.TestCase): # pylint: disable=too-many-public-methods258 """259 Tests for the update_tags() function.260 """261 CONFIG = update_test_lifecycle.Config(262 test_fail_rates=update_test_lifecycle.Rates(263 acceptable=0, unacceptable=1), task_fail_rates=update_test_lifecycle.Rates(264 acceptable=0, unacceptable=1), variant_fail_rates=update_test_lifecycle.Rates(265 acceptable=0, unacceptable=1), distro_fail_rates=update_test_lifecycle.Rates(266 acceptable=0, unacceptable=1), reliable_min_runs=2,267 reliable_time_period=datetime.timedelta(days=1), unreliable_min_runs=2,268 unreliable_time_period=datetime.timedelta(days=1))269 ENTRY = test_failures.ReportEntry(test="jstests/core/all.js", task="jsCore_WT",270 variant="linux-64", distro="rhel62", start_date=datetime.date(271 2017, 6, 3), end_date=datetime.date(2017, 6, 3),272 num_pass=0, num_fail=0)273 def assert_has_only_js_tests(self, lifecycle):274 """275 Raises an AssertionError exception if 'lifecycle' is not of the following form:276 selector:277 js_test:278 ...279 """280 self.assertIn("selector", lifecycle.raw)281 self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))282 self.assertIn("js_test", lifecycle.raw["selector"])283 self.assertEqual(1, len(lifecycle.raw["selector"]), msg=str(lifecycle.raw))284 return lifecycle.raw["selector"]["js_test"]285 def transition_from_reliable_to_unreliable(self, config, expected_tags):286 """287 Tests that update_tags() tags a formerly reliable combination as being unreliable.288 """289 initial_tags = collections.OrderedDict()290 lifecycle = ci_tags.TagsConfig.from_dict(291 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))292 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)293 self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle))294 tests = ["jstests/core/all.js"]295 report = test_failures.Report([296 self.ENTRY._replace(num_pass=0, num_fail=1),297 self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),298 self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),299 self.ENTRY._replace(num_pass=1, num_fail=0),300 self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),301 ])302 update_test_lifecycle.validate_config(config)303 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)304 updated_tags = self.assert_has_only_js_tests(lifecycle)305 self.assertEqual(updated_tags, expected_tags)306 def test_transition_test_from_reliable_to_unreliable(self):307 """308 Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.309 """310 config = self.CONFIG._replace(311 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1))312 self.transition_from_reliable_to_unreliable(config,313 collections.OrderedDict([314 ("jstests/core/all.js", ["unreliable"]),315 ]))316 def test_transition_task_from_reliable_to_unreliable(self):317 """318 Tests that update_tags() tags a formerly reliable (test, task) combination as being319 unreliable.320 """321 config = self.CONFIG._replace(322 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1))323 self.transition_from_reliable_to_unreliable(config,324 collections.OrderedDict([325 ("jstests/core/all.js",326 ["unreliable|jsCore_WT"]),327 ]))328 def test_transition_variant_from_reliable_to_unreliable(self):329 """330 Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being331 unreliable.332 """333 config = self.CONFIG._replace(334 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1))335 self.transition_from_reliable_to_unreliable(config,336 collections.OrderedDict([337 ("jstests/core/all.js",338 ["unreliable|jsCore_WT|linux-64"]),339 ]))340 def test_transition_distro_from_reliable_to_unreliable(self):341 """342 Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination343 as being unreliable.344 """345 config = self.CONFIG._replace(346 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))347 self.transition_from_reliable_to_unreliable(config,348 collections.OrderedDict([349 ("jstests/core/all.js",350 ["unreliable|jsCore_WT|linux-64|rhel62"]),351 ]))352 def test_transition_from_reliable_to_unreliable(self):353 """354 Tests that update_tags() tags multiple formerly reliable combination as being unreliable.355 """356 config = self.CONFIG._replace(357 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),358 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),359 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),360 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))361 self.transition_from_reliable_to_unreliable(config,362 collections.OrderedDict([363 ("jstests/core/all.js", [364 "unreliable",365 "unreliable|jsCore_WT",366 "unreliable|jsCore_WT|linux-64",367 "unreliable|jsCore_WT|linux-64|rhel62",368 ]),369 ]))370 def transition_from_unreliable_to_reliable(self, config, initial_tags):371 """372 Tests that update_tags() untags a formerly unreliable combination after it has become373 reliable again.374 """375 lifecycle = ci_tags.TagsConfig.from_dict(376 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))377 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)378 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))379 tests = ["jstests/core/all.js"]380 report = test_failures.Report([381 self.ENTRY._replace(num_pass=1, num_fail=0),382 self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),383 self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),384 self.ENTRY._replace(num_pass=0, num_fail=1),385 self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),386 ])387 update_test_lifecycle.validate_config(config)388 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)389 updated_tags = self.assert_has_only_js_tests(lifecycle)390 self.assertEqual(updated_tags, collections.OrderedDict())391 def test_non_running_in_reliable_period_is_reliable(self):392 """393 Tests that tests that have a failure rate above the unacceptable rate during the unreliable394 period but haven't run during the reliable period are marked as reliable.395 """396 # Unreliable period is 2 days: 2017-06-03 to 2017-06-04.397 # Reliable period is 1 day: 2016-06-04.398 reliable_period_date = datetime.date(2017, 6, 4)399 config = self.CONFIG._replace(400 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),401 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),402 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),403 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),404 unreliable_time_period=datetime.timedelta(days=2))405 tests = ["jstests/core/all.js"]406 initial_tags = collections.OrderedDict([407 ("jstests/core/all.js", [408 "unreliable",409 "unreliable|jsCore_WT",410 "unreliable|jsCore_WT|linux-64",411 "unreliable|jsCore_WT|linux-64|rhel62",412 ]),413 ])414 lifecycle = ci_tags.TagsConfig.from_dict(415 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))416 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)417 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))418 # The test did not run on the reliable period on linux-64.419 report = test_failures.Report([420 # Failing.421 self.ENTRY._replace(num_pass=0, num_fail=2),422 # Passing on a different variant.423 self.ENTRY._replace(start_date=reliable_period_date, end_date=reliable_period_date,424 num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),425 ])426 update_test_lifecycle.validate_config(config)427 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)428 updated_tags = self.assert_has_only_js_tests(lifecycle)429 # The tags for variant and distro have been removed.430 self.assertEqual(updated_tags,431 collections.OrderedDict([("jstests/core/all.js",432 ["unreliable", "unreliable|jsCore_WT"])]))433 def test_non_running_at_all_is_reliable(self):434 """435 Tests that tests that are tagged as unreliable but no longer running (either during the436 reliable or the unreliable period) have their tags removed.437 """438 config = self.CONFIG439 tests = ["jstests/core/all.js", "jstests/core/all2.js"]440 initial_tags = collections.OrderedDict([441 ("jstests/core/all2.js", [442 "unreliable",443 "unreliable|jsCore_WT",444 "unreliable|jsCore_WT|linux-64",445 "unreliable|jsCore_WT|linux-64|rhel62",446 ]),447 ])448 lifecycle = ci_tags.TagsConfig.from_dict(449 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))450 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)451 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))452 # all2.js did not run at all453 report = test_failures.Report([self.ENTRY])454 update_test_lifecycle.validate_config(config)455 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)456 updated_tags = self.assert_has_only_js_tests(lifecycle)457 # The tags for variant and distro have been removed.458 self.assertEqual(updated_tags, collections.OrderedDict([]))459 def test_transition_test_from_unreliable_to_reliable(self):460 """461 Tests that update_tags() untags a formerly unreliable (test,) combination after it has462 become reliable again.463 """464 config = self.CONFIG._replace(465 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9))466 self.transition_from_unreliable_to_reliable(config,467 collections.OrderedDict([468 ("jstests/core/all.js", ["unreliable"]),469 ]))470 def test_transition_task_from_unreliable_to_reliable(self):471 """472 Tests that update_tags() untags a formerly unreliable (test, task) combination after it has473 become reliable again.474 """475 config = self.CONFIG._replace(476 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9))477 self.transition_from_unreliable_to_reliable(config,478 collections.OrderedDict([479 ("jstests/core/all.js",480 ["unreliable|jsCore_WT"]),481 ]))482 def test_transition_variant_from_unreliable_to_reliable(self):483 """484 Tests that update_tags() untags a formerly unreliable (test, task, variant) combination485 after it has become reliable again.486 """487 config = self.CONFIG._replace(488 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9))489 self.transition_from_unreliable_to_reliable(config,490 collections.OrderedDict([491 ("jstests/core/all.js",492 ["unreliable|jsCore_WT|linux-64"]),493 ]))494 def test_transition_distro_from_unreliable_to_reliable(self):495 """496 Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)497 combination after it has become reliable again.498 """499 config = self.CONFIG._replace(500 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))501 self.transition_from_unreliable_to_reliable(config,502 collections.OrderedDict([503 ("jstests/core/all.js",504 ["unreliable|jsCore_WT|linux-64|rhel62"]),505 ]))506 def test_transition_from_unreliable_to_reliable(self):507 """508 Tests that update_tags() untags multiple formerly unreliable combination after it has become509 reliable again.510 """511 config = self.CONFIG._replace(512 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),513 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),514 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),515 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))516 self.transition_from_unreliable_to_reliable(config,517 collections.OrderedDict([518 ("jstests/core/all.js", [519 "unreliable",520 "unreliable|jsCore_WT",521 "unreliable|jsCore_WT|linux-64",522 "unreliable|jsCore_WT|linux-64|rhel62",523 ]),524 ]))525 def test_remain_reliable(self):526 """527 Tests that update_tags() preserves the absence of tags for reliable combinations.528 """529 config = self.CONFIG._replace(530 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),531 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),532 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),533 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))534 initial_tags = collections.OrderedDict()535 lifecycle = ci_tags.TagsConfig.from_dict(536 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))537 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)538 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))539 tests = ["jstests/core/all.js"]540 report = test_failures.Report([541 self.ENTRY._replace(num_pass=1, num_fail=0),542 self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),543 self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),544 self.ENTRY._replace(num_pass=0, num_fail=1),545 self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),546 ])547 update_test_lifecycle.validate_config(config)548 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)549 updated_tags = self.assert_has_only_js_tests(lifecycle)550 self.assertEqual(updated_tags, initial_tags)551 def test_remain_unreliable(self):552 """553 Tests that update_tags() preserves the tags for unreliable combinations.554 """555 config = self.CONFIG._replace(556 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),557 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),558 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),559 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))560 initial_tags = collections.OrderedDict([561 ("jstests/core/all.js", [562 "unreliable",563 "unreliable|jsCore_WT",564 "unreliable|jsCore_WT|linux-64",565 "unreliable|jsCore_WT|linux-64|rhel62",566 ]),567 ])568 lifecycle = ci_tags.TagsConfig.from_dict(569 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))570 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)571 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))572 tests = ["jstests/core/all.js"]573 report = test_failures.Report([574 self.ENTRY._replace(num_pass=0, num_fail=1),575 self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),576 self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),577 self.ENTRY._replace(num_pass=1, num_fail=0),578 self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),579 ])580 update_test_lifecycle.validate_config(config)581 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)582 updated_tags = self.assert_has_only_js_tests(lifecycle)583 self.assertEqual(updated_tags, initial_tags)584 def test_obeys_reliable_min_runs(self):585 """586 Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.587 """588 config = self.CONFIG._replace(589 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),590 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),591 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),592 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9),593 reliable_min_runs=100)594 self.transition_from_unreliable_to_reliable(config,595 collections.OrderedDict([596 ("jstests/core/all.js", [597 "unreliable",598 "unreliable|jsCore_WT",599 "unreliable|jsCore_WT|linux-64",600 "unreliable|jsCore_WT|linux-64|rhel62",601 ]),602 ]))603 def test_obeys_reliable_time_period(self):604 """605 Tests that update_tags() ignores passes from before 'reliable_time_period'.606 """607 config = self.CONFIG._replace(608 test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),609 task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),610 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),611 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))612 initial_tags = collections.OrderedDict()613 lifecycle = ci_tags.TagsConfig.from_dict(614 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))615 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)616 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))617 tests = ["jstests/core/all.js"]618 report = test_failures.Report([619 self.ENTRY._replace(620 start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)),621 end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=1,622 num_fail=0),623 self.ENTRY._replace(624 start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)),625 end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=1,626 num_fail=0),627 self.ENTRY._replace(num_pass=0, num_fail=1),628 self.ENTRY._replace(num_pass=0, num_fail=1),629 self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),630 self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),631 self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),632 ])633 update_test_lifecycle.validate_config(config)634 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)635 updated_tags = self.assert_has_only_js_tests(lifecycle)636 self.assertEqual(updated_tags,637 collections.OrderedDict([638 ("jstests/core/all.js", [639 "unreliable",640 "unreliable|jsCore_WT",641 "unreliable|jsCore_WT|linux-64",642 "unreliable|jsCore_WT|linux-64|rhel62",643 ]),644 ]))645 def test_obeys_unreliable_min_runs(self):646 """647 Tests that update_tags() only considers a test unreliable if it has more than648 'unreliable_min_runs'.649 """650 config = self.CONFIG._replace(651 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),652 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),653 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),654 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),655 unreliable_min_runs=100)656 initial_tags = collections.OrderedDict()657 lifecycle = ci_tags.TagsConfig.from_dict(658 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))659 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)660 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))661 tests = ["jstests/core/all.js"]662 report = test_failures.Report([663 self.ENTRY._replace(num_pass=0, num_fail=1),664 self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"),665 self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"),666 self.ENTRY._replace(num_pass=1, num_fail=0),667 self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"),668 ])669 update_test_lifecycle.validate_config(config)670 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)671 updated_tags = self.assert_has_only_js_tests(lifecycle)672 self.assertEqual(updated_tags, initial_tags)673 def test_obeys_unreliable_time_period(self):674 """675 Tests that update_tags() ignores failures from before 'unreliable_time_period'.676 """677 config = self.CONFIG._replace(678 test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),679 task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),680 variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),681 distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))682 initial_tags = collections.OrderedDict([683 ("jstests/core/all.js", [684 "unreliable",685 "unreliable|jsCore_WT",686 "unreliable|jsCore_WT|linux-64",687 "unreliable|jsCore_WT|linux-64|rhel62",688 ]),689 ])690 lifecycle = ci_tags.TagsConfig.from_dict(691 dict(selector=dict(js_test=copy.deepcopy(initial_tags))))692 summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle)693 self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))694 tests = ["jstests/core/all.js"]695 report = test_failures.Report([696 self.ENTRY._replace(697 start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)),698 end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=0,699 num_fail=1),700 self.ENTRY._replace(701 start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)),702 end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=0,703 num_fail=1),704 self.ENTRY._replace(num_pass=1, num_fail=0),705 self.ENTRY._replace(num_pass=1, num_fail=0),706 self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"),707 self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"),708 self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"),709 ])710 update_test_lifecycle.validate_config(config)711 update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests)712 updated_tags = self.assert_has_only_js_tests(lifecycle)713 self.assertEqual(updated_tags, collections.OrderedDict())714class TestCombinationHelpers(unittest.TestCase):715 def test_from_entry(self):716 entry = test_failures._ReportEntry("testA", "taskA", "variantA", "distroA",717 datetime.date.today(), datetime.date.today(), 0, 0)718 combination = update_test_lifecycle._test_combination_from_entry(719 entry, test_failures.Report.TEST)720 self.assertEqual(combination, ("testA", ))721 combination = update_test_lifecycle._test_combination_from_entry(722 entry, test_failures.Report.TEST_TASK)...

Full Screen

Full Screen

gen_build_yaml.py

Source:gen_build_yaml.py Github

copy

Full Screen

...21 'fullstack includes_proxy dns_resolver name_resolution secure platforms ci_mac tracing exclude_configs exclude_iomgrs large_writes enables_compression supports_compression is_inproc is_http2 supports_proxy_auth supports_write_buffering client_channel')22default_unsecure_fixture_options = FixtureOptions(23 True, False, True, True, False, ['windows', 'linux', 'mac', 'posix'],24 True, False, [], [], True, False, True, False, True, False, True, True)25socketpair_unsecure_fixture_options = default_unsecure_fixture_options._replace(26 fullstack=False, dns_resolver=False, client_channel=False)27default_secure_fixture_options = default_unsecure_fixture_options._replace(28 secure=True)29uds_fixture_options = default_unsecure_fixture_options._replace(30 dns_resolver=False, platforms=['linux', 'mac', 'posix'],31 exclude_iomgrs=['uv'])32local_fixture_options = default_secure_fixture_options._replace(33 dns_resolver=False, platforms=['linux', 'mac', 'posix'],34 exclude_iomgrs=['uv'])35fd_unsecure_fixture_options = default_unsecure_fixture_options._replace(36 dns_resolver=False, fullstack=False, platforms=['linux', 'mac', 'posix'],37 exclude_iomgrs=['uv'], client_channel=False)38inproc_fixture_options = default_secure_fixture_options._replace(39 dns_resolver=False, fullstack=False, name_resolution=False,40 supports_compression=False, is_inproc=True, is_http2=False,41 supports_write_buffering=False, client_channel=False)42# maps fixture name to whether it requires the security library43END2END_FIXTURES = {44 'h2_compress': default_unsecure_fixture_options._replace(enables_compression=True),45 'h2_census': default_unsecure_fixture_options,46 # This cmake target is disabled for now because it depends on OpenCensus,47 # which is Bazel-only.48 # 'h2_load_reporting': default_unsecure_fixture_options,49 'h2_fakesec': default_secure_fixture_options._replace(ci_mac=False),50 'h2_fd': fd_unsecure_fixture_options,51 'h2_full': default_unsecure_fixture_options,52 'h2_full+pipe': default_unsecure_fixture_options._replace(53 platforms=['linux'], exclude_iomgrs=['uv']),54 'h2_full+trace': default_unsecure_fixture_options._replace(tracing=True),55 'h2_full+workarounds': default_unsecure_fixture_options,56 'h2_http_proxy': default_unsecure_fixture_options._replace(57 ci_mac=False, exclude_iomgrs=['uv'], supports_proxy_auth=True),58 'h2_oauth2': default_secure_fixture_options._replace(59 ci_mac=False, exclude_iomgrs=['uv']),60 'h2_proxy': default_unsecure_fixture_options._replace(61 includes_proxy=True, ci_mac=False, exclude_iomgrs=['uv']),62 'h2_sockpair_1byte': socketpair_unsecure_fixture_options._replace(63 ci_mac=False, exclude_configs=['msan'], large_writes=False,64 exclude_iomgrs=['uv']),65 'h2_sockpair': socketpair_unsecure_fixture_options._replace(66 ci_mac=False, exclude_iomgrs=['uv']),67 'h2_sockpair+trace': socketpair_unsecure_fixture_options._replace(68 ci_mac=False, tracing=True, large_writes=False, exclude_iomgrs=['uv']),69 'h2_ssl': default_secure_fixture_options,70 'h2_ssl_cred_reload': default_secure_fixture_options,71 'h2_spiffe': default_secure_fixture_options,72 'h2_local_uds': local_fixture_options,73 'h2_local_ipv4': local_fixture_options,74 'h2_local_ipv6': local_fixture_options,75 'h2_ssl_proxy': default_secure_fixture_options._replace(76 includes_proxy=True, ci_mac=False, exclude_iomgrs=['uv']),77 'h2_uds': uds_fixture_options,78 'inproc': inproc_fixture_options79}80TestOptions = collections.namedtuple(81 'TestOptions',82 'needs_fullstack needs_dns needs_names proxyable secure traceable cpu_cost exclude_iomgrs large_writes flaky allows_compression needs_compression exclude_inproc needs_http2 needs_proxy_auth needs_write_buffering needs_client_channel')83default_test_options = TestOptions(84 False, False, False, True, False, True, 1.0, [], False, False, True,85 False, False, False, False, False, False)86connectivity_test_options = default_test_options._replace(87 needs_fullstack=True)88LOWCPU = 0.189# maps test names to options90END2END_TESTS = {91 'authority_not_supported': default_test_options,92 'bad_hostname': default_test_options._replace(needs_names=True),93 'bad_ping': connectivity_test_options._replace(proxyable=False),94 'binary_metadata': default_test_options._replace(cpu_cost=LOWCPU),95 'resource_quota_server': default_test_options._replace(96 large_writes=True, proxyable=False, allows_compression=False),97 'call_creds': default_test_options._replace(secure=True),98 'cancel_after_accept': default_test_options._replace(cpu_cost=LOWCPU),99 'cancel_after_client_done': default_test_options._replace(cpu_cost=LOWCPU),100 'cancel_after_invoke': default_test_options._replace(cpu_cost=LOWCPU),101 'cancel_after_round_trip': default_test_options._replace(cpu_cost=LOWCPU),102 'cancel_before_invoke': default_test_options._replace(cpu_cost=LOWCPU),103 'cancel_in_a_vacuum': default_test_options._replace(cpu_cost=LOWCPU),104 'cancel_with_status': default_test_options._replace(cpu_cost=LOWCPU),105 'compressed_payload': default_test_options._replace(proxyable=False,106 needs_compression=True),107 'connectivity': connectivity_test_options._replace(needs_names=True,108 proxyable=False, cpu_cost=LOWCPU, exclude_iomgrs=['uv']),109 'channelz': default_test_options,110 'default_host': default_test_options._replace(111 needs_fullstack=True, needs_dns=True, needs_names=True),112 'call_host_override': default_test_options._replace(113 needs_fullstack=True, needs_dns=True, needs_names=True),114 'disappearing_server': connectivity_test_options._replace(flaky=True,115 needs_names=True),116 'empty_batch': default_test_options._replace(cpu_cost=LOWCPU),117 'filter_causes_close': default_test_options._replace(cpu_cost=LOWCPU),118 'filter_call_init_fails': default_test_options,119 'filter_context': default_test_options,120 'filter_latency': default_test_options._replace(cpu_cost=LOWCPU),121 'filter_status_code': default_test_options._replace(cpu_cost=LOWCPU),122 'graceful_server_shutdown': default_test_options._replace(123 cpu_cost=LOWCPU, exclude_inproc=True),124 'hpack_size': default_test_options._replace(proxyable=False,125 traceable=False,126 cpu_cost=LOWCPU),127 'high_initial_seqno': default_test_options._replace(cpu_cost=LOWCPU),128 'idempotent_request': default_test_options,129 'invoke_large_request': default_test_options,130 'keepalive_timeout': default_test_options._replace(proxyable=False,131 cpu_cost=LOWCPU,132 needs_http2=True),133 'large_metadata': default_test_options,134 'max_concurrent_streams': default_test_options._replace(135 proxyable=False, cpu_cost=LOWCPU, exclude_inproc=True),136 'max_connection_age': default_test_options._replace(cpu_cost=LOWCPU,137 exclude_inproc=True),138 'max_connection_idle': connectivity_test_options._replace(139 proxyable=False, exclude_iomgrs=['uv'], cpu_cost=LOWCPU),140 'max_message_length': default_test_options._replace(cpu_cost=LOWCPU),141 'negative_deadline': default_test_options,142 'no_error_on_hotpath': default_test_options._replace(proxyable=False),143 'no_logging': default_test_options._replace(traceable=False),144 'no_op': default_test_options,145 'payload': default_test_options,146 # This cmake target is disabled for now because it depends on OpenCensus,147 # which is Bazel-only.148 # 'load_reporting_hook': default_test_options,149 'ping_pong_streaming': default_test_options._replace(cpu_cost=LOWCPU),150 'ping': connectivity_test_options._replace(proxyable=False,151 cpu_cost=LOWCPU),152 'proxy_auth': default_test_options._replace(needs_proxy_auth=True),153 'registered_call': default_test_options,154 'request_with_flags': default_test_options._replace(155 proxyable=False, cpu_cost=LOWCPU),156 'request_with_payload': default_test_options._replace(cpu_cost=LOWCPU),157 # TODO(roth): Remove proxyable=False for all retry tests once we158 # have a way for the proxy to propagate the fact that trailing159 # metadata is available when initial metadata is returned.160 # See https://github.com/grpc/grpc/issues/14467 for context.161 'retry': default_test_options._replace(cpu_cost=LOWCPU,162 needs_client_channel=True,163 proxyable=False),164 'retry_cancellation': default_test_options._replace(165 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),166 'retry_disabled': default_test_options._replace(cpu_cost=LOWCPU,167 needs_client_channel=True,168 proxyable=False),169 'retry_exceeds_buffer_size_in_initial_batch': default_test_options._replace(170 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),171 'retry_exceeds_buffer_size_in_subsequent_batch':172 default_test_options._replace(cpu_cost=LOWCPU,173 needs_client_channel=True,174 proxyable=False),175 'retry_non_retriable_status': default_test_options._replace(176 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),177 'retry_non_retriable_status_before_recv_trailing_metadata_started':178 default_test_options._replace(179 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),180 'retry_recv_initial_metadata': default_test_options._replace(181 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),182 'retry_recv_message': default_test_options._replace(183 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),184 'retry_server_pushback_delay': default_test_options._replace(185 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),186 'retry_server_pushback_disabled': default_test_options._replace(187 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),188 'retry_streaming': default_test_options._replace(cpu_cost=LOWCPU,189 needs_client_channel=True,190 proxyable=False),191 'retry_streaming_after_commit': default_test_options._replace(192 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),193 'retry_streaming_succeeds_before_replay_finished':194 default_test_options._replace(cpu_cost=LOWCPU,195 needs_client_channel=True,196 proxyable=False),197 'retry_throttled': default_test_options._replace(cpu_cost=LOWCPU,198 needs_client_channel=True,199 proxyable=False),200 'retry_too_many_attempts': default_test_options._replace(201 cpu_cost=LOWCPU, needs_client_channel=True, proxyable=False),202 'server_finishes_request': default_test_options._replace(cpu_cost=LOWCPU),203 'shutdown_finishes_calls': default_test_options._replace(cpu_cost=LOWCPU),204 'shutdown_finishes_tags': default_test_options._replace(cpu_cost=LOWCPU),205 'simple_cacheable_request': default_test_options._replace(cpu_cost=LOWCPU),206 'stream_compression_compressed_payload': default_test_options._replace(207 proxyable=False, exclude_inproc=True),208 'stream_compression_payload': default_test_options._replace(209 exclude_inproc=True),210 'stream_compression_ping_pong_streaming': default_test_options._replace(211 exclude_inproc=True),212 'simple_delayed_request': connectivity_test_options,213 'simple_metadata': default_test_options,214 'simple_request': default_test_options,215 'streaming_error_response': default_test_options._replace(cpu_cost=LOWCPU),216 'trailing_metadata': default_test_options,217 'workaround_cronet_compression': default_test_options,218 'write_buffering': default_test_options._replace(219 cpu_cost=LOWCPU, needs_write_buffering=True),220 'write_buffering_at_end': default_test_options._replace(221 cpu_cost=LOWCPU, needs_write_buffering=True),222}223def compatible(f, t):224 if END2END_TESTS[t].needs_fullstack:225 if not END2END_FIXTURES[f].fullstack:226 return False227 if END2END_TESTS[t].needs_dns:228 if not END2END_FIXTURES[f].dns_resolver:229 return False230 if END2END_TESTS[t].needs_names:231 if not END2END_FIXTURES[f].name_resolution:232 return False233 if not END2END_TESTS[t].proxyable:234 if END2END_FIXTURES[f].includes_proxy:...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run localstack automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful