Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use test_failures method in assertpy

Best Python code snippet using assertpy_python

test_lifecycle_test_failures.py

Source:test_lifecycle_test_failures.py

1"""2Tests for buildscripts/test_failures.py.3"""4from __future__ import absolute_import5import datetime6import unittest7from buildscripts import lifecycle_test_failures as test_failures8# pylint: disable=invalid-name,missing-docstring,protected-access9class TestReportEntry(unittest.TestCase):10    """11    Tests for the test_failures.ReportEntry class.12    """13    ENTRY = test_failures.ReportEntry(test="jstests/core/all.js", task="jsCore_WT",14                                      variant="linux-64", distro="rhel62", start_date=datetime.date(15                                          2017, 6, 3), end_date=datetime.date(2017, 6, 3),16                                      num_pass=0, num_fail=0)17    def test_fail_rate(self):18        """19        Tests for the test_failures.ReportEntry.fail_rate property.20        """21        entry = self.ENTRY._replace(num_pass=0, num_fail=1)22        self.assertEqual(1, entry.fail_rate)23        entry = self.ENTRY._replace(num_pass=9, num_fail=1)24        self.assertAlmostEqual(0.1, entry.fail_rate)25        # Verify that we don't attempt to divide by zero.26        entry = self.ENTRY._replace(num_pass=0, num_fail=0)27        self.assertEqual(0, entry.fail_rate)28    def test_week_start_date_with_sunday(self):29        """30        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week31        specified as different forms of the string "Sunday".32        """33        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))34        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("sunday"))35        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("Sunday"))36        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("SUNDAY"))37        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))38        self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))39        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))40        self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))41    def test_week_start_date_with_monday(self):42        """43        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week44        specified as different forms of the string "Monday".45        """46        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))47        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))48        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("Monday"))49        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("MONDAY"))50        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))51        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))52        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))53        self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))54        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 6))55        self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))56    def test_week_start_date_with_date(self):57        """58        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week59        specified as a datetime.date() value.60        """61        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))62        date = datetime.date(2017, 5, 21)63        self.assertEqual(6, date.weekday(), "2017 May 21 is a Sunday")64        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date(date))65        date = datetime.date(2017, 5, 22)66        self.assertEqual(0, date.weekday(), "2017 May 22 is a Monday")67        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date(date))68        date = datetime.date(2017, 6, 6)69        self.assertEqual(1, date.weekday(), "2017 Jun 06 is a Tuesday")70        self.assertEqual(datetime.date(2017, 5, 30), entry.week_start_date(date))71        date = datetime.date(2017, 6, 9)72        self.assertEqual(4, date.weekday(), "2017 Jun 09 is a Friday")73        self.assertEqual(datetime.date(2017, 6, 2), entry.week_start_date(date))74        date = datetime.date(2017, 6, 3)75        self.assertEqual(5, date.weekday(), "2017 Jun 03 is a Saturday")76        self.assertEqual(datetime.date(2017, 6, 3), entry.week_start_date(date))77    def test_sum_combines_test_results(self):78        """79        Tests for test_failures.ReportEntry.sum() that verify the start_date, end_date, num_pass,80        and num_fail attributes are accumulated correctly.81        """82        entry1 = self.ENTRY._replace(83            start_date=datetime.date(2017, 6, 1), end_date=datetime.date(2017, 6, 1), num_pass=1,84            num_fail=0)85        entry2 = self.ENTRY._replace(86            start_date=datetime.date(2017, 6, 2), end_date=datetime.date(2017, 6, 2), num_pass=0,87            num_fail=3)88        entry3 = self.ENTRY._replace(89            start_date=datetime.date(2017, 6, 3), end_date=datetime.date(2017, 6, 3), num_pass=0,90            num_fail=0)91        entry4 = self.ENTRY._replace(92            start_date=datetime.date(2017, 6, 4), end_date=datetime.date(2017, 6, 4), num_pass=2,93            num_fail=2)94        entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])95        entry_1432 = test_failures.ReportEntry.sum([entry1, entry4, entry3, entry2])96        entry_124 = test_failures.ReportEntry.sum([entry1, entry2, entry4])97        entry_13 = test_failures.ReportEntry.sum([entry1, entry3])98        entry_42 = test_failures.ReportEntry.sum([entry4, entry2])99        self.assertEqual(datetime.date(2017, 6, 1), entry_1234.start_date)100        self.assertEqual(datetime.date(2017, 6, 4), entry_1234.end_date)101        self.assertEqual(3, entry_1234.num_pass)102        self.assertEqual(5, entry_1234.num_fail)103        self.assertEqual(entry_1234, entry_1432, "order of arguments shouldn't matter")104        self.assertEqual(entry_1234, entry_124, "entry3 didn't have any test executions")105        self.assertEqual(datetime.date(2017, 6, 1), entry_13.start_date)106        self.assertEqual(datetime.date(2017, 6, 3), entry_13.end_date)107        self.assertEqual(1, entry_13.num_pass)108        self.assertEqual(0, entry_13.num_fail)109        self.assertEqual(datetime.date(2017, 6, 2), entry_42.start_date)110        self.assertEqual(datetime.date(2017, 6, 4), entry_42.end_date)111        self.assertEqual(2, entry_42.num_pass)112        self.assertEqual(5, entry_42.num_fail)113    def test_sum_combines_test_info(self):114        """115        Tests for test_failures.ReportEntry.sum() that verify the test, task, variant, and distro116        attributes are accumulated correctly.117        """118        entry1 = self.ENTRY._replace(test="jstests/core/all.js", task="jsCore_WT",119                                     variant="linux-64", distro="rhel62")120        entry2 = self.ENTRY._replace(test="jstests/core/all.js", task="jsCore_WT",121                                     variant="linux-64", distro="rhel55")122        entry3 = self.ENTRY._replace(test="jstests/core/all2.js", task="jsCore_WT",123                                     variant="linux-64-debug", distro="rhel62")124        entry4 = self.ENTRY._replace(test="jstests/core/all.js", task="jsCore",125                                     variant="linux-64-debug", distro="rhel62")126        entry_12 = test_failures.ReportEntry.sum([entry1, entry2])127        self.assertEqual("jstests/core/all.js", entry_12.test)128        self.assertEqual("jsCore_WT", entry_12.task)129        self.assertEqual("linux-64", entry_12.variant)130        self.assertIsInstance(entry_12.distro, test_failures.Wildcard)131        entry_123 = test_failures.ReportEntry.sum([entry1, entry2, entry3])132        self.assertIsInstance(entry_123.test, test_failures.Wildcard)133        self.assertEqual("jsCore_WT", entry_123.task)134        self.assertIsInstance(entry_123.variant, test_failures.Wildcard)135        self.assertIsInstance(entry_123.distro, test_failures.Wildcard)136        entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])137        self.assertIsInstance(entry_1234.test, test_failures.Wildcard)138        self.assertIsInstance(entry_1234.task, test_failures.Wildcard)139        self.assertIsInstance(entry_1234.variant, test_failures.Wildcard)140        self.assertIsInstance(entry_1234.distro, test_failures.Wildcard)141        entry_34 = test_failures.ReportEntry.sum([entry3, entry4])142        self.assertIsInstance(entry_34.test, test_failures.Wildcard)143        self.assertIsInstance(entry_34.task, test_failures.Wildcard)144        self.assertEqual("linux-64-debug", entry_34.variant)145        self.assertEqual("rhel62", entry_34.distro)146class TestReportSummarization(unittest.TestCase):147    """148    Tests for test_failures.Report.summarize_by().149    """150    ENTRY = test_failures.ReportEntry(test="jstests/core/all.js", task="jsCore_WT",151                                      variant="linux-64", distro="rhel62", start_date=datetime.date(152                                          2017, 6, 3), end_date=datetime.date(2017, 6, 3),153                                      num_pass=0, num_fail=0)154    ENTRIES = [155        ENTRY._replace(156            start_date=datetime.date(2017, 6, 3), end_date=datetime.date(2017, 6, 3), num_pass=1,157            num_fail=0),158        ENTRY._replace(task="jsCore", start_date=datetime.date(2017, 6, 5), end_date=datetime.date(159            2017, 6, 5), num_pass=0, num_fail=1),160        ENTRY._replace(161            start_date=datetime.date(2017, 6, 10), end_date=datetime.date(2017, 6, 10), num_pass=1,162            num_fail=0),163        # The following entry is intentionally not in timestamp order to verify that the164        # 'time_period' parameter becomes part of the sort in summarize_by().165        ENTRY._replace(166            start_date=datetime.date(2017, 6, 9), end_date=datetime.date(2017, 6, 9), num_pass=1,167            num_fail=0),168        ENTRY._replace(distro="rhel55", start_date=datetime.date(2017, 6, 10),169                       end_date=datetime.date(2017, 6, 10), num_pass=0, num_fail=1),170        ENTRY._replace(test="jstests/core/all2.js", start_date=datetime.date(2017, 6, 10),171                       end_date=datetime.date(2017, 6, 10), num_pass=1, num_fail=0),172        ENTRY._replace(variant="linux-64-debug", start_date=datetime.date(2017, 6, 17),173                       end_date=datetime.date(2017, 6, 17), num_pass=0, num_fail=1),174    ]175    def test_group_all_by_test_task_variant_distro(self):176        """177        Tests that summarize_by() correctly accumulates all unique combinations of178        (test, task, variant, distro).179        """180        report = test_failures.Report(self.ENTRIES)181        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT_DISTRO)182        self.assertEqual(5, len(summed_entries))183        self.assertEqual(summed_entries[0],184                         self.ENTRY._replace(185                             task="jsCore",186                             start_date=datetime.date(2017, 6, 5),187                             end_date=datetime.date(2017, 6, 5),188                             num_pass=0,189                             num_fail=1,190                         ))191        self.assertEqual(summed_entries[1],192                         self.ENTRY._replace(193                             distro="rhel55",194                             start_date=datetime.date(2017, 6, 10),195                             end_date=datetime.date(2017, 6, 10),196                             num_pass=0,197                             num_fail=1,198                         ))199        self.assertEqual(summed_entries[2],200                         self.ENTRY._replace(201                             start_date=datetime.date(2017, 6, 3),202                             end_date=datetime.date(2017, 6, 10),203                             num_pass=3,204                             num_fail=0,205                         ))206        self.assertEqual(summed_entries[3],207                         self.ENTRY._replace(208                             variant="linux-64-debug",209                             start_date=datetime.date(2017, 6, 17),210                             end_date=datetime.date(2017, 6, 17),211                             num_pass=0,212                             num_fail=1,213                         ))214        self.assertEqual(summed_entries[4],215                         self.ENTRY._replace(216                             test="jstests/core/all2.js",217                             start_date=datetime.date(2017, 6, 10),218                             end_date=datetime.date(2017, 6, 10),219                             num_pass=1,220                             num_fail=0,221                         ))222    def test_group_all_by_test_task_variant(self):223        """224        Tests that summarize_by() correctly accumulates all unique combinations of225        (test, task, variant).226        """227        report = test_failures.Report(self.ENTRIES)228        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT)229        self.assertEqual(4, len(summed_entries))230        self.assertEqual(summed_entries[0],231                         self.ENTRY._replace(232                             task="jsCore",233                             start_date=datetime.date(2017, 6, 5),234                             end_date=datetime.date(2017, 6, 5),235                             num_pass=0,236                             num_fail=1,237                         ))238        self.assertEqual(summed_entries[1],239                         self.ENTRY._replace(240                             distro=test_failures.Wildcard("distros"),241                             start_date=datetime.date(2017, 6, 3),242                             end_date=datetime.date(2017, 6, 10),243                             num_pass=3,244                             num_fail=1,245                         ))246        self.assertEqual(summed_entries[2],247                         self.ENTRY._replace(248                             variant="linux-64-debug",249                             start_date=datetime.date(2017, 6, 17),250                             end_date=datetime.date(2017, 6, 17),251                             num_pass=0,252                             num_fail=1,253                         ))254        self.assertEqual(summed_entries[3],255                         self.ENTRY._replace(256                             test="jstests/core/all2.js",257                             start_date=datetime.date(2017, 6, 10),258                             end_date=datetime.date(2017, 6, 10),259                             num_pass=1,260                             num_fail=0,261                         ))262    def test_group_all_by_test_task(self):263        """264        Tests that summarize_by() correctly accumulates all unique combinations of (test, task).265        """266        report = test_failures.Report(self.ENTRIES)267        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK)268        self.assertEqual(3, len(summed_entries))269        self.assertEqual(summed_entries[0],270                         self.ENTRY._replace(271                             task="jsCore",272                             start_date=datetime.date(2017, 6, 5),273                             end_date=datetime.date(2017, 6, 5),274                             num_pass=0,275                             num_fail=1,276                         ))277        self.assertEqual(summed_entries[1],278                         self.ENTRY._replace(279                             variant=test_failures.Wildcard("variants"),280                             distro=test_failures.Wildcard("distros"),281                             start_date=datetime.date(2017, 6, 3),282                             end_date=datetime.date(2017, 6, 17),283                             num_pass=3,284                             num_fail=2,285                         ))286        self.assertEqual(summed_entries[2],287                         self.ENTRY._replace(288                             test="jstests/core/all2.js",289                             start_date=datetime.date(2017, 6, 10),290                             end_date=datetime.date(2017, 6, 10),291                             num_pass=1,292                             num_fail=0,293                         ))294    def test_group_all_by_test(self):295        """296        Tests that summarize_by() correctly accumulates all unique combinations of (test,).297        """298        report = test_failures.Report(self.ENTRIES)299        summed_entries = report.summarize_by(test_failures.Report.TEST)300        self.assertEqual(2, len(summed_entries))301        self.assertEqual(summed_entries[0],302                         self.ENTRY._replace(303                             task=test_failures.Wildcard("tasks"),304                             variant=test_failures.Wildcard("variants"),305                             distro=test_failures.Wildcard("distros"),306                             start_date=datetime.date(2017, 6, 3),307                             end_date=datetime.date(2017, 6, 17),308                             num_pass=3,309                             num_fail=3,310                         ))311        self.assertEqual(summed_entries[1],312                         self.ENTRY._replace(313                             test="jstests/core/all2.js",314                             start_date=datetime.date(2017, 6, 10),315                             end_date=datetime.date(2017, 6, 10),316                             num_pass=1,317                             num_fail=0,318                         ))319    def test_group_all_by_variant_task(self):320        """321        Tests that summarize_by() correctly accumulates all unique combinations of (variant, task).322        """323        report = test_failures.Report(self.ENTRIES)324        summed_entries = report.summarize_by(["variant", "task"])325        self.assertEqual(3, len(summed_entries))326        self.assertEqual(summed_entries[0],327                         self.ENTRY._replace(328                             task="jsCore",329                             start_date=datetime.date(2017, 6, 5),330                             end_date=datetime.date(2017, 6, 5),331                             num_pass=0,332                             num_fail=1,333                         ))334        self.assertEqual(summed_entries[1],335                         self.ENTRY._replace(336                             test=test_failures.Wildcard("tests"),337                             distro=test_failures.Wildcard("distros"),338                             start_date=datetime.date(2017, 6, 3),339                             end_date=datetime.date(2017, 6, 10),340                             num_pass=4,341                             num_fail=1,342                         ))343        self.assertEqual(summed_entries[2],344                         self.ENTRY._replace(345                             variant="linux-64-debug",346                             start_date=datetime.date(2017, 6, 17),347                             end_date=datetime.date(2017, 6, 17),348                             num_pass=0,349                             num_fail=1,350                         ))351    def test_group_weekly_by_test_starting_on_sunday(self):352        """353        Tests that summarize_by() correctly accumulates by week when the beginning of the week is354        specified as the string "sunday".355        """356        report = test_failures.Report(self.ENTRIES)357        summed_entries = report.summarize_by(test_failures.Report.TEST,358                                             time_period=test_failures.Report.WEEKLY,359                                             start_day_of_week=test_failures.Report.SUNDAY)360        self.assertEqual(4, len(summed_entries))361        self.assertEqual(summed_entries[0],362                         self.ENTRY._replace(363                             start_date=datetime.date(2017, 6, 3),364                             end_date=datetime.date(2017, 6, 3),365                             num_pass=1,366                             num_fail=0,367                         ))368        self.assertEqual(summed_entries[1],369                         self.ENTRY._replace(370                             task=test_failures.Wildcard("tasks"),371                             distro=test_failures.Wildcard("distros"),372                             start_date=datetime.date(2017, 6, 4),373                             end_date=datetime.date(2017, 6, 10),374                             num_pass=2,375                             num_fail=2,376                         ))377        self.assertEqual(summed_entries[2],378                         self.ENTRY._replace(379                             variant="linux-64-debug",380                             start_date=datetime.date(2017, 6, 11),381                             end_date=datetime.date(2017, 6, 17),382                             num_pass=0,383                             num_fail=1,384                         ))385        self.assertEqual(summed_entries[3],386                         self.ENTRY._replace(387                             test="jstests/core/all2.js",388                             start_date=datetime.date(2017, 6, 4),389                             end_date=datetime.date(2017, 6, 10),390                             num_pass=1,391                             num_fail=0,392                         ))393    def test_group_weekly_by_test_starting_on_monday(self):394        """395        Tests that summarize_by() correctly accumulates by week when the beginning of the week is396        specified as the string "monday".397        """398        report = test_failures.Report(self.ENTRIES)399        summed_entries = report.summarize_by(test_failures.Report.TEST,400                                             time_period=test_failures.Report.WEEKLY,401                                             start_day_of_week=test_failures.Report.MONDAY)402        self.assertEqual(4, len(summed_entries))403        self.assertEqual(summed_entries[0],404                         self.ENTRY._replace(405                             start_date=datetime.date(2017, 6, 3),406                             end_date=datetime.date(2017, 6, 4),407                             num_pass=1,408                             num_fail=0,409                         ))410        self.assertEqual(summed_entries[1],411                         self.ENTRY._replace(412                             task=test_failures.Wildcard("tasks"),413                             distro=test_failures.Wildcard("distros"),414                             start_date=datetime.date(2017, 6, 5),415                             end_date=datetime.date(2017, 6, 11),416                             num_pass=2,417                             num_fail=2,418                         ))419        self.assertEqual(summed_entries[2],420                         self.ENTRY._replace(421                             variant="linux-64-debug",422                             start_date=datetime.date(2017, 6, 12),423                             end_date=datetime.date(2017, 6, 17),424                             num_pass=0,425                             num_fail=1,426                         ))427        self.assertEqual(summed_entries[3],428                         self.ENTRY._replace(429                             test="jstests/core/all2.js",430                             start_date=datetime.date(2017, 6, 5),431                             end_date=datetime.date(2017, 6, 11),432                             num_pass=1,433                             num_fail=0,434                         ))435    def test_group_weekly_by_test_starting_on_date(self):436        """437        Tests that summarize_by() correctly accumulates by week when the beginning of the week is438        specified as a datetime.date() value.439        """440        date = datetime.date(2017, 6, 7)441        self.assertEqual(2, date.weekday(), "2017 Jun 07 is a Wednesday")442        report = test_failures.Report(self.ENTRIES)443        summed_entries = report.summarize_by(test_failures.Report.TEST,444                                             time_period=test_failures.Report.WEEKLY,445                                             start_day_of_week=date)446        self.assertEqual(4, len(summed_entries))447        self.assertEqual(summed_entries[0],448                         self.ENTRY._replace(449                             task=test_failures.Wildcard("tasks"),450                             start_date=datetime.date(2017, 6, 3),451                             end_date=datetime.date(2017, 6, 6),452                             num_pass=1,453                             num_fail=1,454                         ))455        self.assertEqual(summed_entries[1],456                         self.ENTRY._replace(457                             distro=test_failures.Wildcard("distros"),458                             start_date=datetime.date(2017, 6, 7),459                             end_date=datetime.date(2017, 6, 13),460                             num_pass=2,461                             num_fail=1,462                         ))463        self.assertEqual(summed_entries[2],464                         self.ENTRY._replace(465                             variant="linux-64-debug",466                             start_date=datetime.date(2017, 6, 14),467                             end_date=datetime.date(2017, 6, 17),468                             num_pass=0,469                             num_fail=1,470                         ))471        self.assertEqual(summed_entries[3],472                         self.ENTRY._replace(473                             test="jstests/core/all2.js",474                             start_date=datetime.date(2017, 6, 7),475                             end_date=datetime.date(2017, 6, 13),476                             num_pass=1,477                             num_fail=0,478                         ))479    def test_group_daily_by_test(self):480        """481        Tests that summarize_by() correctly accumulates by day.482        """483        report = test_failures.Report(self.ENTRIES)484        summed_entries = report.summarize_by(test_failures.Report.TEST,485                                             time_period=test_failures.Report.DAILY)486        self.assertEqual(6, len(summed_entries))487        self.assertEqual(summed_entries[0],488                         self.ENTRY._replace(489                             start_date=datetime.date(2017, 6, 3),490                             end_date=datetime.date(2017, 6, 3),491                             num_pass=1,492                             num_fail=0,493                         ))494        self.assertEqual(summed_entries[1],495                         self.ENTRY._replace(496                             task="jsCore",497                             start_date=datetime.date(2017, 6, 5),498                             end_date=datetime.date(2017, 6, 5),499                             num_pass=0,500                             num_fail=1,501                         ))502        self.assertEqual(summed_entries[2],503                         self.ENTRY._replace(504                             start_date=datetime.date(2017, 6, 9),505                             end_date=datetime.date(2017, 6, 9),506                             num_pass=1,507                             num_fail=0,508                         ))509        self.assertEqual(summed_entries[3],510                         self.ENTRY._replace(511                             distro=test_failures.Wildcard("distros"),512                             start_date=datetime.date(2017, 6, 10),513                             end_date=datetime.date(2017, 6, 10),514                             num_pass=1,515                             num_fail=1,516                         ))517        self.assertEqual(summed_entries[4],518                         self.ENTRY._replace(519                             variant="linux-64-debug",520                             start_date=datetime.date(2017, 6, 17),521                             end_date=datetime.date(2017, 6, 17),522                             num_pass=0,523                             num_fail=1,524                         ))525        self.assertEqual(summed_entries[5],526                         self.ENTRY._replace(527                             test="jstests/core/all2.js",528                             start_date=datetime.date(2017, 6, 10),529                             end_date=datetime.date(2017, 6, 10),530                             num_pass=1,531                             num_fail=0,532                         ))533    def test_group_4days_by_test(self):534        """535        Tests that summarize_by() correctly accumulates by multiple days.536        """537        report = test_failures.Report(self.ENTRIES)538        summed_entries = report.summarize_by(test_failures.Report.TEST,539                                             time_period=datetime.timedelta(days=4))540        self.assertEqual(4, len(summed_entries))541        self.assertEqual(summed_entries[0],542                         self.ENTRY._replace(543                             task=test_failures.Wildcard("tasks"),544                             start_date=datetime.date(2017, 6, 3),545                             end_date=datetime.date(2017, 6, 6),546                             num_pass=1,547                             num_fail=1,548                         ))549        self.assertEqual(summed_entries[1],550                         self.ENTRY._replace(551                             distro=test_failures.Wildcard("distros"),552                             start_date=datetime.date(2017, 6, 7),553                             end_date=datetime.date(2017, 6, 10),554                             num_pass=2,555                             num_fail=1,556                         ))557        self.assertEqual(summed_entries[2],558                         self.ENTRY._replace(559                             variant="linux-64-debug",560                             start_date=datetime.date(2017, 6, 15),561                             end_date=datetime.date(2017, 6, 17),562                             num_pass=0,563                             num_fail=1,564                         ))565        self.assertEqual(summed_entries[3],566                         self.ENTRY._replace(567                             test="jstests/core/all2.js",568                             start_date=datetime.date(2017, 6, 7),569                             end_date=datetime.date(2017, 6, 10),570                             num_pass=1,571                             num_fail=0,572                         ))573    def test_group_9days_by_test(self):574        """575        Tests that summarize_by() correctly accumulates by multiple days, including time periods576        greater than 1 week.577        """578        report = test_failures.Report(self.ENTRIES)579        summed_entries = report.summarize_by(test_failures.Report.TEST,580                                             time_period=datetime.timedelta(days=9))581        self.assertEqual(3, len(summed_entries))582        self.assertEqual(summed_entries[0],583                         self.ENTRY._replace(584                             task=test_failures.Wildcard("tasks"),585                             distro=test_failures.Wildcard("distros"),586                             start_date=datetime.date(2017, 6, 3),587                             end_date=datetime.date(2017, 6, 11),588                             num_pass=3,589                             num_fail=2,590                         ))591        self.assertEqual(summed_entries[1],592                         self.ENTRY._replace(593                             variant="linux-64-debug",594                             start_date=datetime.date(2017, 6, 12),595                             end_date=datetime.date(2017, 6, 17),596                             num_pass=0,597                             num_fail=1,598                         ))599        self.assertEqual(summed_entries[2],600                         self.ENTRY._replace(601                             test="jstests/core/all2.js",602                             start_date=datetime.date(2017, 6, 3),603                             end_date=datetime.date(2017, 6, 11),604                             num_pass=1,605                             num_fail=0,...

test_test_failures.py

Source:test_test_failures.py

1"""2Tests for buildscripts/test_failures.py.3"""4from __future__ import absolute_import5import datetime6import unittest7from buildscripts import test_failures8class TestReportEntry(unittest.TestCase):9    """10    Tests for the test_failures.ReportEntry class.11    """12    ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",13                                      task="jsCore_WT",14                                      variant="linux-64",15                                      distro="rhel62",16                                      start_date=datetime.date(2017, 6, 3),17                                      end_date=datetime.date(2017, 6, 3),18                                      num_pass=0,19                                      num_fail=0)20    def test_fail_rate(self):21        """22        Tests for the test_failures.ReportEntry.fail_rate property.23        """24        entry = self.ENTRY._replace(num_pass=0, num_fail=1)25        self.assertEqual(1, entry.fail_rate)26        entry = self.ENTRY._replace(num_pass=9, num_fail=1)27        self.assertAlmostEqual(0.1, entry.fail_rate)28        # Verify that we don't attempt to divide by zero.29        entry = self.ENTRY._replace(num_pass=0, num_fail=0)30        self.assertEqual(0, entry.fail_rate)31    def test_week_start_date_with_sunday(self):32        """33        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week34        specified as different forms of the string "Sunday".35        """36        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))37        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("sunday"))38        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("Sunday"))39        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date("SUNDAY"))40        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))41        self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))42        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))43        self.assertEqual(datetime.date(2017, 6, 4), entry.week_start_date("sunday"))44    def test_week_start_date_with_monday(self):45        """46        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week47        specified as different forms of the string "Monday".48        """49        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))50        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))51        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("Monday"))52        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("MONDAY"))53        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4))54        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date("monday"))55        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 5))56        self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))57        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 6))58        self.assertEqual(datetime.date(2017, 6, 5), entry.week_start_date("monday"))59    def test_week_start_date_with_date(self):60        """61        Tests for test_failures.ReportEntry.week_start_date() with the beginning of the week62        specified as a datetime.date() value.63        """64        entry = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3))65        date = datetime.date(2017, 5, 21)66        self.assertEqual(6, date.weekday(), "2017 May 21 is a Sunday")67        self.assertEqual(datetime.date(2017, 5, 28), entry.week_start_date(date))68        date = datetime.date(2017, 5, 22)69        self.assertEqual(0, date.weekday(), "2017 May 22 is a Monday")70        self.assertEqual(datetime.date(2017, 5, 29), entry.week_start_date(date))71        date = datetime.date(2017, 6, 6)72        self.assertEqual(1, date.weekday(), "2017 Jun 06 is a Tuesday")73        self.assertEqual(datetime.date(2017, 5, 30), entry.week_start_date(date))74        date = datetime.date(2017, 6, 9)75        self.assertEqual(4, date.weekday(), "2017 Jun 09 is a Friday")76        self.assertEqual(datetime.date(2017, 6, 2), entry.week_start_date(date))77        date = datetime.date(2017, 6, 3)78        self.assertEqual(5, date.weekday(), "2017 Jun 03 is a Saturday")79        self.assertEqual(datetime.date(2017, 6, 3), entry.week_start_date(date))80    def test_sum_combines_test_results(self):81        """82        Tests for test_failures.ReportEntry.sum() that verify the start_date, end_date, num_pass,83        and num_fail attributes are accumulated correctly.84        """85        entry1 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 1),86                                     end_date=datetime.date(2017, 6, 1),87                                     num_pass=1,88                                     num_fail=0)89        entry2 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 2),90                                     end_date=datetime.date(2017, 6, 2),91                                     num_pass=0,92                                     num_fail=3)93        entry3 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 3),94                                     end_date=datetime.date(2017, 6, 3),95                                     num_pass=0,96                                     num_fail=0)97        entry4 = self.ENTRY._replace(start_date=datetime.date(2017, 6, 4),98                                     end_date=datetime.date(2017, 6, 4),99                                     num_pass=2,100                                     num_fail=2)101        entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])102        entry_1432 = test_failures.ReportEntry.sum([entry1, entry4, entry3, entry2])103        entry_124 = test_failures.ReportEntry.sum([entry1, entry2, entry4])104        entry_13 = test_failures.ReportEntry.sum([entry1, entry3])105        entry_42 = test_failures.ReportEntry.sum([entry4, entry2])106        self.assertEqual(datetime.date(2017, 6, 1), entry_1234.start_date)107        self.assertEqual(datetime.date(2017, 6, 4), entry_1234.end_date)108        self.assertEqual(3, entry_1234.num_pass)109        self.assertEqual(5, entry_1234.num_fail)110        self.assertEqual(entry_1234, entry_1432, "order of arguments shouldn't matter")111        self.assertEqual(entry_1234, entry_124, "entry3 didn't have any test executions")112        self.assertEqual(datetime.date(2017, 6, 1), entry_13.start_date)113        self.assertEqual(datetime.date(2017, 6, 3), entry_13.end_date)114        self.assertEqual(1, entry_13.num_pass)115        self.assertEqual(0, entry_13.num_fail)116        self.assertEqual(datetime.date(2017, 6, 2), entry_42.start_date)117        self.assertEqual(datetime.date(2017, 6, 4), entry_42.end_date)118        self.assertEqual(2, entry_42.num_pass)119        self.assertEqual(5, entry_42.num_fail)120    def test_sum_combines_test_info(self):121        """122        Tests for test_failures.ReportEntry.sum() that verify the test, task, variant, and distro123        attributes are accumulated correctly.124        """125        entry1 = self.ENTRY._replace(test="jstests/core/all.js",126                                     task="jsCore_WT",127                                     variant="linux-64",128                                     distro="rhel62")129        entry2 = self.ENTRY._replace(test="jstests/core/all.js",130                                     task="jsCore_WT",131                                     variant="linux-64",132                                     distro="rhel55")133        entry3 = self.ENTRY._replace(test="jstests/core/all2.js",134                                     task="jsCore_WT",135                                     variant="linux-64-debug",136                                     distro="rhel62")137        entry4 = self.ENTRY._replace(test="jstests/core/all.js",138                                     task="jsCore",139                                     variant="linux-64-debug",140                                     distro="rhel62")141        entry_12 = test_failures.ReportEntry.sum([entry1, entry2])142        self.assertEqual("jstests/core/all.js", entry_12.test)143        self.assertEqual("jsCore_WT", entry_12.task)144        self.assertEqual("linux-64", entry_12.variant)145        self.assertIsInstance(entry_12.distro, test_failures.Wildcard)146        entry_123 = test_failures.ReportEntry.sum([entry1, entry2, entry3])147        self.assertIsInstance(entry_123.test, test_failures.Wildcard)148        self.assertEqual("jsCore_WT", entry_123.task)149        self.assertIsInstance(entry_123.variant, test_failures.Wildcard)150        self.assertIsInstance(entry_123.distro, test_failures.Wildcard)151        entry_1234 = test_failures.ReportEntry.sum([entry1, entry2, entry3, entry4])152        self.assertIsInstance(entry_1234.test, test_failures.Wildcard)153        self.assertIsInstance(entry_1234.task, test_failures.Wildcard)154        self.assertIsInstance(entry_1234.variant, test_failures.Wildcard)155        self.assertIsInstance(entry_1234.distro, test_failures.Wildcard)156        entry_34 = test_failures.ReportEntry.sum([entry3, entry4])157        self.assertIsInstance(entry_34.test, test_failures.Wildcard)158        self.assertIsInstance(entry_34.task, test_failures.Wildcard)159        self.assertEqual("linux-64-debug", entry_34.variant)160        self.assertEqual("rhel62", entry_34.distro)161class TestReportSummarization(unittest.TestCase):162    """163    Tests for test_failures.Report.summarize_by().164    """165    ENTRY = test_failures.ReportEntry(test="jstests/core/all.js",166                                      task="jsCore_WT",167                                      variant="linux-64",168                                      distro="rhel62",169                                      start_date=datetime.date(2017, 6, 3),170                                      end_date=datetime.date(2017, 6, 3),171                                      num_pass=0,172                                      num_fail=0)173    ENTRIES = [174        ENTRY._replace(start_date=datetime.date(2017, 6, 3),175                       end_date=datetime.date(2017, 6, 3),176                       num_pass=1,177                       num_fail=0),178        ENTRY._replace(task="jsCore",179                       start_date=datetime.date(2017, 6, 5),180                       end_date=datetime.date(2017, 6, 5),181                       num_pass=0,182                       num_fail=1),183        ENTRY._replace(start_date=datetime.date(2017, 6, 10),184                       end_date=datetime.date(2017, 6, 10),185                       num_pass=1,186                       num_fail=0),187        # The following entry is intentionally not in timestamp order to verify that the188        # 'time_period' parameter becomes part of the sort in summarize_by().189        ENTRY._replace(start_date=datetime.date(2017, 6, 9),190                       end_date=datetime.date(2017, 6, 9),191                       num_pass=1,192                       num_fail=0),193        ENTRY._replace(distro="rhel55",194                       start_date=datetime.date(2017, 6, 10),195                       end_date=datetime.date(2017, 6, 10),196                       num_pass=0,197                       num_fail=1),198        ENTRY._replace(test="jstests/core/all2.js",199                       start_date=datetime.date(2017, 6, 10),200                       end_date=datetime.date(2017, 6, 10),201                       num_pass=1,202                       num_fail=0),203        ENTRY._replace(variant="linux-64-debug",204                       start_date=datetime.date(2017, 6, 17),205                       end_date=datetime.date(2017, 6, 17),206                       num_pass=0,207                       num_fail=1),208    ]209    def test_group_all_by_test_task_variant_distro(self):210        """211        Tests that summarize_by() correctly accumulates all unique combinations of212        (test, task, variant, distro).213        """214        report = test_failures.Report(self.ENTRIES)215        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT_DISTRO)216        self.assertEqual(5, len(summed_entries))217        self.assertEqual(summed_entries[0], self.ENTRY._replace(218            task="jsCore",219            start_date=datetime.date(2017, 6, 5),220            end_date=datetime.date(2017, 6, 5),221            num_pass=0,222            num_fail=1,223        ))224        self.assertEqual(summed_entries[1], self.ENTRY._replace(225            distro="rhel55",226            start_date=datetime.date(2017, 6, 10),227            end_date=datetime.date(2017, 6, 10),228            num_pass=0,229            num_fail=1,230        ))231        self.assertEqual(summed_entries[2], self.ENTRY._replace(232            start_date=datetime.date(2017, 6, 3),233            end_date=datetime.date(2017, 6, 10),234            num_pass=3,235            num_fail=0,236        ))237        self.assertEqual(summed_entries[3], self.ENTRY._replace(238            variant="linux-64-debug",239            start_date=datetime.date(2017, 6, 17),240            end_date=datetime.date(2017, 6, 17),241            num_pass=0,242            num_fail=1,243        ))244        self.assertEqual(summed_entries[4], self.ENTRY._replace(245            test="jstests/core/all2.js",246            start_date=datetime.date(2017, 6, 10),247            end_date=datetime.date(2017, 6, 10),248            num_pass=1,249            num_fail=0,250        ))251    def test_group_all_by_test_task_variant(self):252        """253        Tests that summarize_by() correctly accumulates all unique combinations of254        (test, task, variant).255        """256        report = test_failures.Report(self.ENTRIES)257        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK_VARIANT)258        self.assertEqual(4, len(summed_entries))259        self.assertEqual(summed_entries[0], self.ENTRY._replace(260            task="jsCore",261            start_date=datetime.date(2017, 6, 5),262            end_date=datetime.date(2017, 6, 5),263            num_pass=0,264            num_fail=1,265        ))266        self.assertEqual(summed_entries[1], self.ENTRY._replace(267            distro=test_failures.Wildcard("distros"),268            start_date=datetime.date(2017, 6, 3),269            end_date=datetime.date(2017, 6, 10),270            num_pass=3,271            num_fail=1,272        ))273        self.assertEqual(summed_entries[2], self.ENTRY._replace(274            variant="linux-64-debug",275            start_date=datetime.date(2017, 6, 17),276            end_date=datetime.date(2017, 6, 17),277            num_pass=0,278            num_fail=1,279        ))280        self.assertEqual(summed_entries[3], self.ENTRY._replace(281            test="jstests/core/all2.js",282            start_date=datetime.date(2017, 6, 10),283            end_date=datetime.date(2017, 6, 10),284            num_pass=1,285            num_fail=0,286        ))287    def test_group_all_by_test_task(self):288        """289        Tests that summarize_by() correctly accumulates all unique combinations of (test, task).290        """291        report = test_failures.Report(self.ENTRIES)292        summed_entries = report.summarize_by(test_failures.Report.TEST_TASK)293        self.assertEqual(3, len(summed_entries))294        self.assertEqual(summed_entries[0], self.ENTRY._replace(295            task="jsCore",296            start_date=datetime.date(2017, 6, 5),297            end_date=datetime.date(2017, 6, 5),298            num_pass=0,299            num_fail=1,300        ))301        self.assertEqual(summed_entries[1], self.ENTRY._replace(302            variant=test_failures.Wildcard("variants"),303            distro=test_failures.Wildcard("distros"),304            start_date=datetime.date(2017, 6, 3),305            end_date=datetime.date(2017, 6, 17),306            num_pass=3,307            num_fail=2,308        ))309        self.assertEqual(summed_entries[2], self.ENTRY._replace(310            test="jstests/core/all2.js",311            start_date=datetime.date(2017, 6, 10),312            end_date=datetime.date(2017, 6, 10),313            num_pass=1,314            num_fail=0,315        ))316    def test_group_all_by_test(self):317        """318        Tests that summarize_by() correctly accumulates all unique combinations of (test,).319        """320        report = test_failures.Report(self.ENTRIES)321        summed_entries = report.summarize_by(test_failures.Report.TEST)322        self.assertEqual(2, len(summed_entries))323        self.assertEqual(summed_entries[0], self.ENTRY._replace(324            task=test_failures.Wildcard("tasks"),325            variant=test_failures.Wildcard("variants"),326            distro=test_failures.Wildcard("distros"),327            start_date=datetime.date(2017, 6, 3),328            end_date=datetime.date(2017, 6, 17),329            num_pass=3,330            num_fail=3,331        ))332        self.assertEqual(summed_entries[1], self.ENTRY._replace(333            test="jstests/core/all2.js",334            start_date=datetime.date(2017, 6, 10),335            end_date=datetime.date(2017, 6, 10),336            num_pass=1,337            num_fail=0,338        ))339    def test_group_all_by_variant_task(self):340        """341        Tests that summarize_by() correctly accumulates all unique combinations of (variant, task).342        """343        report = test_failures.Report(self.ENTRIES)344        summed_entries = report.summarize_by(["variant", "task"])345        self.assertEqual(3, len(summed_entries))346        self.assertEqual(summed_entries[0], self.ENTRY._replace(347            task="jsCore",348            start_date=datetime.date(2017, 6, 5),349            end_date=datetime.date(2017, 6, 5),350            num_pass=0,351            num_fail=1,352        ))353        self.assertEqual(summed_entries[1], self.ENTRY._replace(354            test=test_failures.Wildcard("tests"),355            distro=test_failures.Wildcard("distros"),356            start_date=datetime.date(2017, 6, 3),357            end_date=datetime.date(2017, 6, 10),358            num_pass=4,359            num_fail=1,360        ))361        self.assertEqual(summed_entries[2], self.ENTRY._replace(362            variant="linux-64-debug",363            start_date=datetime.date(2017, 6, 17),364            end_date=datetime.date(2017, 6, 17),365            num_pass=0,366            num_fail=1,367        ))368    def test_group_weekly_by_test_starting_on_sunday(self):369        """370        Tests that summarize_by() correctly accumulates by week when the beginning of the week is371        specified as the string "sunday".372        """373        report = test_failures.Report(self.ENTRIES)374        summed_entries = report.summarize_by(test_failures.Report.TEST,375                                             time_period=test_failures.Report.WEEKLY,376                                             start_day_of_week=test_failures.Report.SUNDAY)377        self.assertEqual(4, len(summed_entries))378        self.assertEqual(summed_entries[0], self.ENTRY._replace(379            start_date=datetime.date(2017, 6, 3),380            end_date=datetime.date(2017, 6, 3),381            num_pass=1,382            num_fail=0,383        ))384        self.assertEqual(summed_entries[1], self.ENTRY._replace(385            task=test_failures.Wildcard("tasks"),386            distro=test_failures.Wildcard("distros"),387            start_date=datetime.date(2017, 6, 4),388            end_date=datetime.date(2017, 6, 10),389            num_pass=2,390            num_fail=2,391        ))392        self.assertEqual(summed_entries[2], self.ENTRY._replace(393            variant="linux-64-debug",394            start_date=datetime.date(2017, 6, 11),395            end_date=datetime.date(2017, 6, 17),396            num_pass=0,397            num_fail=1,398        ))399        self.assertEqual(summed_entries[3], self.ENTRY._replace(400            test="jstests/core/all2.js",401            start_date=datetime.date(2017, 6, 4),402            end_date=datetime.date(2017, 6, 10),403            num_pass=1,404            num_fail=0,405        ))406    def test_group_weekly_by_test_starting_on_monday(self):407        """408        Tests that summarize_by() correctly accumulates by week when the beginning of the week is409        specified as the string "monday".410        """411        report = test_failures.Report(self.ENTRIES)412        summed_entries = report.summarize_by(test_failures.Report.TEST,413                                             time_period=test_failures.Report.WEEKLY,414                                             start_day_of_week=test_failures.Report.MONDAY)415        self.assertEqual(4, len(summed_entries))416        self.assertEqual(summed_entries[0], self.ENTRY._replace(417            start_date=datetime.date(2017, 6, 3),418            end_date=datetime.date(2017, 6, 4),419            num_pass=1,420            num_fail=0,421        ))422        self.assertEqual(summed_entries[1], self.ENTRY._replace(423            task=test_failures.Wildcard("tasks"),424            distro=test_failures.Wildcard("distros"),425            start_date=datetime.date(2017, 6, 5),426            end_date=datetime.date(2017, 6, 11),427            num_pass=2,428            num_fail=2,429        ))430        self.assertEqual(summed_entries[2], self.ENTRY._replace(431            variant="linux-64-debug",432            start_date=datetime.date(2017, 6, 12),433            end_date=datetime.date(2017, 6, 17),434            num_pass=0,435            num_fail=1,436        ))437        self.assertEqual(summed_entries[3], self.ENTRY._replace(438            test="jstests/core/all2.js",439            start_date=datetime.date(2017, 6, 5),440            end_date=datetime.date(2017, 6, 11),441            num_pass=1,442            num_fail=0,443        ))444    def test_group_weekly_by_test_starting_on_date(self):445        """446        Tests that summarize_by() correctly accumulates by week when the beginning of the week is447        specified as a datetime.date() value.448        """449        date = datetime.date(2017, 6, 7)450        self.assertEqual(2, date.weekday(), "2017 Jun 07 is a Wednesday")451        report = test_failures.Report(self.ENTRIES)452        summed_entries = report.summarize_by(test_failures.Report.TEST,453                                             time_period=test_failures.Report.WEEKLY,454                                             start_day_of_week=date)455        self.assertEqual(4, len(summed_entries))456        self.assertEqual(summed_entries[0], self.ENTRY._replace(457            task=test_failures.Wildcard("tasks"),458            start_date=datetime.date(2017, 6, 3),459            end_date=datetime.date(2017, 6, 6),460            num_pass=1,461            num_fail=1,462        ))463        self.assertEqual(summed_entries[1], self.ENTRY._replace(464            distro=test_failures.Wildcard("distros"),465            start_date=datetime.date(2017, 6, 7),466            end_date=datetime.date(2017, 6, 13),467            num_pass=2,468            num_fail=1,469        ))470        self.assertEqual(summed_entries[2], self.ENTRY._replace(471            variant="linux-64-debug",472            start_date=datetime.date(2017, 6, 14),473            end_date=datetime.date(2017, 6, 17),474            num_pass=0,475            num_fail=1,476        ))477        self.assertEqual(summed_entries[3], self.ENTRY._replace(478            test="jstests/core/all2.js",479            start_date=datetime.date(2017, 6, 7),480            end_date=datetime.date(2017, 6, 13),481            num_pass=1,482            num_fail=0,483        ))484    def test_group_daily_by_test(self):485        """486        Tests that summarize_by() correctly accumulates by day.487        """488        report = test_failures.Report(self.ENTRIES)489        summed_entries = report.summarize_by(test_failures.Report.TEST,490                                             time_period=test_failures.Report.DAILY)491        self.assertEqual(6, len(summed_entries))492        self.assertEqual(summed_entries[0], self.ENTRY._replace(493            start_date=datetime.date(2017, 6, 3),494            end_date=datetime.date(2017, 6, 3),495            num_pass=1,496            num_fail=0,497        ))498        self.assertEqual(summed_entries[1], self.ENTRY._replace(499            task="jsCore",500            start_date=datetime.date(2017, 6, 5),501            end_date=datetime.date(2017, 6, 5),502            num_pass=0,503            num_fail=1,504        ))505        self.assertEqual(summed_entries[2], self.ENTRY._replace(506            start_date=datetime.date(2017, 6, 9),507            end_date=datetime.date(2017, 6, 9),508            num_pass=1,509            num_fail=0,510        ))511        self.assertEqual(summed_entries[3], self.ENTRY._replace(512            distro=test_failures.Wildcard("distros"),513            start_date=datetime.date(2017, 6, 10),514            end_date=datetime.date(2017, 6, 10),515            num_pass=1,516            num_fail=1,517        ))518        self.assertEqual(summed_entries[4], self.ENTRY._replace(519            variant="linux-64-debug",520            start_date=datetime.date(2017, 6, 17),521            end_date=datetime.date(2017, 6, 17),522            num_pass=0,523            num_fail=1,524        ))525        self.assertEqual(summed_entries[5], self.ENTRY._replace(526            test="jstests/core/all2.js",527            start_date=datetime.date(2017, 6, 10),528            end_date=datetime.date(2017, 6, 10),529            num_pass=1,530            num_fail=0,531        ))532    def test_group_4days_by_test(self):533        """534        Tests that summarize_by() correctly accumulates by multiple days.535        """536        report = test_failures.Report(self.ENTRIES)537        summed_entries = report.summarize_by(test_failures.Report.TEST,538                                             time_period=datetime.timedelta(days=4))539        self.assertEqual(4, len(summed_entries))540        self.assertEqual(summed_entries[0], self.ENTRY._replace(541            task=test_failures.Wildcard("tasks"),542            start_date=datetime.date(2017, 6, 3),543            end_date=datetime.date(2017, 6, 6),544            num_pass=1,545            num_fail=1,546        ))547        self.assertEqual(summed_entries[1], self.ENTRY._replace(548            distro=test_failures.Wildcard("distros"),549            start_date=datetime.date(2017, 6, 7),550            end_date=datetime.date(2017, 6, 10),551            num_pass=2,552            num_fail=1,553        ))554        self.assertEqual(summed_entries[2], self.ENTRY._replace(555            variant="linux-64-debug",556            start_date=datetime.date(2017, 6, 15),557            end_date=datetime.date(2017, 6, 17),558            num_pass=0,559            num_fail=1,560        ))561        self.assertEqual(summed_entries[3], self.ENTRY._replace(562            test="jstests/core/all2.js",563            start_date=datetime.date(2017, 6, 7),564            end_date=datetime.date(2017, 6, 10),565            num_pass=1,566            num_fail=0,567        ))568    def test_group_9days_by_test(self):569        """570        Tests that summarize_by() correctly accumulates by multiple days, including time periods571        greater than 1 week.572        """573        report = test_failures.Report(self.ENTRIES)574        summed_entries = report.summarize_by(test_failures.Report.TEST,575                                             time_period=datetime.timedelta(days=9))576        self.assertEqual(3, len(summed_entries))577        self.assertEqual(summed_entries[0], self.ENTRY._replace(578            task=test_failures.Wildcard("tasks"),579            distro=test_failures.Wildcard("distros"),580            start_date=datetime.date(2017, 6, 3),581            end_date=datetime.date(2017, 6, 11),582            num_pass=3,583            num_fail=2,584        ))585        self.assertEqual(summed_entries[1], self.ENTRY._replace(586            variant="linux-64-debug",587            start_date=datetime.date(2017, 6, 12),588            end_date=datetime.date(2017, 6, 17),589            num_pass=0,590            num_fail=1,591        ))592        self.assertEqual(summed_entries[2], self.ENTRY._replace(593            test="jstests/core/all2.js",594            start_date=datetime.date(2017, 6, 3),595            end_date=datetime.date(2017, 6, 11),596            num_pass=1,597            num_fail=0,...

gdb_pretty_printer_test.py

Source:gdb_pretty_printer_test.py

1#===----------------------------------------------------------------------===##2#3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4# See https://llvm.org/LICENSE.txt for license information.5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6#7#===----------------------------------------------------------------------===##8"""Commands used to automate testing gdb pretty printers.9This script is part of a larger framework to test gdb pretty printers. It10runs the program, detects test cases, checks them, and prints results.11See gdb_pretty_printer_test.sh.cpp on how to write a test case.12"""13from __future__ import print_function14import re15import gdb16import sys17test_failures = 018class CheckResult(gdb.Command):19    def __init__(self):20        super(CheckResult, self).__init__(21            "print_and_compare", gdb.COMMAND_DATA)22    def invoke(self, arg, from_tty):23        try:24            # Stack frame is:25            # 0. StopForDebugger26            # 1. ComparePrettyPrintToChars or ComparePrettyPrintToRegex27            # 2. TestCase28            compare_frame = gdb.newest_frame().older()29            testcase_frame = compare_frame.older()30            test_loc = testcase_frame.find_sal()31            # Use interactive commands in the correct context to get the pretty32            # printed version33            value_str = self._get_value_string(compare_frame, testcase_frame)34            # Ignore the convenience variable name and newline35            value = value_str[value_str.find("= ") + 2:-1]36            gdb.newest_frame().select()37            expectation_val = compare_frame.read_var("expectation")38            check_literal = expectation_val.string(encoding="utf-8")39            if "PrettyPrintToRegex" in compare_frame.name():40                test_fails = not re.match(check_literal, value)41            else:42                test_fails = value != check_literal43            if test_fails:44                global test_failures45                print("FAIL: " + test_loc.symtab.filename +46                      ":" + str(test_loc.line))47                print("GDB printed:")48                print("   " + repr(value))49                print("Value should match:")50                print("   " + repr(check_literal))51                test_failures += 152            else:53                print("PASS: " + test_loc.symtab.filename +54                      ":" + str(test_loc.line))55        except RuntimeError as e:56            # At this point, lots of different things could be wrong, so don't try to57            # recover or figure it out. Don't exit either, because then it's58            # impossible debug the framework itself.59            print("FAIL: Something is wrong in the test framework.")60            print(str(e))61            test_failures += 162    def _get_value_string(self, compare_frame, testcase_frame):63        compare_frame.select()64        if "ComparePrettyPrint" in compare_frame.name():65            s = gdb.execute("p value", to_string=True)66        else:67            value_str = str(compare_frame.read_var("value"))68            clean_expression_str = value_str.strip("'\"")69            testcase_frame.select()70            s = gdb.execute("p " + clean_expression_str, to_string=True)71        if sys.version_info.major == 2:72            return s.decode("utf-8")73        return s74def exit_handler(event=None):75    global test_failures76    if test_failures:77        print("FAILED %d cases" % test_failures)78    exit(test_failures)79# Start code executed at load time80# Disable terminal paging81gdb.execute("set height 0")82gdb.execute("set python print-stack full")83test_failures = 084CheckResult()85test_bp = gdb.Breakpoint("StopForDebugger")86test_bp.enabled = True87test_bp.silent = True88test_bp.commands = "print_and_compare\ncontinue"89# "run" won't return if the program exits; ensure the script regains control.90gdb.events.exited.connect(exit_handler)91gdb.execute("run")92# If the program didn't exit, something went wrong, but we don't93# know what. Fail on exit.94test_failures += 1...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.