How to use fail_fast method in hypothesis

Best Python code snippet using hypothesis

googletest-failfast-unittest.py

Source:googletest-failfast-unittest.py

1#!/usr/bin/env python2#3# Copyright 2020 Google Inc. All Rights Reserved.4#5# Redistribution and use in source and binary forms, with or without6# modification, are permitted provided that the following conditions are7# met:8#9#     * Redistributions of source code must retain the above copyright10# notice, this list of conditions and the following disclaimer.11#     * Redistributions in binary form must reproduce the above12# copyright notice, this list of conditions and the following disclaimer13# in the documentation and/or other materials provided with the14# distribution.15#     * Neither the name of Google Inc. nor the names of its16# contributors may be used to endorse or promote products derived from17# this software without specific prior written permission.18#19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.30"""Unit test for Google Test fail_fast.31A user can specify if a Google Test program should continue test execution32after a test failure via the GTEST_FAIL_FAST environment variable or the33--gtest_fail_fast flag. The default value of the flag can also be changed34by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.35This script tests such functionality by invoking googletest-failfast-unittest_36(a program written with Google Test) with different environments and command37line flags.38"""39import os40import gtest_test_utils41# Constants.42# Bazel testbridge environment variable for fail fast43BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'44# The environment variable for specifying fail fast.45FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'46# The command line flag for specifying fail fast.47FAIL_FAST_FLAG = 'gtest_fail_fast'48# The command line flag to run disabled tests.49RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'50# The command line flag for specifying a filter.51FILTER_FLAG = 'gtest_filter'52# Command to run the googletest-failfast-unittest_ program.53COMMAND = gtest_test_utils.GetTestExecutablePath(54    'googletest-failfast-unittest_')55# The command line flag to tell Google Test to output the list of tests it56# will run.57LIST_TESTS_FLAG = '--gtest_list_tests'58# Indicates whether Google Test supports death tests.59SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(60    [COMMAND, LIST_TESTS_FLAG]).output61# Utilities.62environ = os.environ.copy()63def SetEnvVar(env_var, value):64  """Sets the env variable to 'value'; unsets it when 'value' is None."""65  if value is not None:66    environ[env_var] = value67  elif env_var in environ:68    del environ[env_var]69def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):70  """Runs the test program and returns its output."""71  args = []72  xml_path = os.path.join(gtest_test_utils.GetTempDir(),73                          '.GTestFailFastUnitTest.xml')74  args += ['--gtest_output=xml:' + xml_path]75  if fail_fast is not None:76    if isinstance(fail_fast, str):77      args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]78    elif fail_fast:79      args += ['--%s' % FAIL_FAST_FLAG]80    else:81      args += ['--no%s' % FAIL_FAST_FLAG]82  if test_suite:83    args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]84  if run_disabled:85    args += ['--%s' % RUN_DISABLED_FLAG]86  txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output87  with open(xml_path) as xml_file:88    return txt_out, xml_file.read()89# The unit test.90class GTestFailFastUnitTest(gtest_test_utils.TestCase):91  """Tests the env variable or the command line flag for fail_fast."""92  def testDefaultBehavior(self):93    """Tests the behavior of not specifying the fail_fast."""94    txt, _ = RunAndReturnOutput()95    self.assertIn('22 FAILED TEST', txt)96  def testGoogletestFlag(self):97    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)98    self.assertIn('1 FAILED TEST', txt)99    self.assertIn('[  SKIPPED ] 3 tests', txt)100    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)101    self.assertIn('4 FAILED TEST', txt)102    self.assertNotIn('[  SKIPPED ]', txt)103  def testGoogletestEnvVar(self):104    """Tests the behavior of specifying fail_fast via Googletest env var."""105    try:106      SetEnvVar(FAIL_FAST_ENV_VAR, '1')107      txt, _ = RunAndReturnOutput('HasSimpleTest')108      self.assertIn('1 FAILED TEST', txt)109      self.assertIn('[  SKIPPED ] 3 tests', txt)110      SetEnvVar(FAIL_FAST_ENV_VAR, '0')111      txt, _ = RunAndReturnOutput('HasSimpleTest')112      self.assertIn('4 FAILED TEST', txt)113      self.assertNotIn('[  SKIPPED ]', txt)114    finally:115      SetEnvVar(FAIL_FAST_ENV_VAR, None)116  def testBazelEnvVar(self):117    """Tests the behavior of specifying fail_fast via Bazel testbridge."""118    try:119      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')120      txt, _ = RunAndReturnOutput('HasSimpleTest')121      self.assertIn('1 FAILED TEST', txt)122      self.assertIn('[  SKIPPED ] 3 tests', txt)123      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')124      txt, _ = RunAndReturnOutput('HasSimpleTest')125      self.assertIn('4 FAILED TEST', txt)126      self.assertNotIn('[  SKIPPED ]', txt)127    finally:128      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)129  def testFlagOverridesEnvVar(self):130    """Tests precedence of flag over env var."""131    try:132      SetEnvVar(FAIL_FAST_ENV_VAR, '0')133      txt, _ = RunAndReturnOutput('HasSimpleTest', True)134      self.assertIn('1 FAILED TEST', txt)135      self.assertIn('[  SKIPPED ] 3 tests', txt)136    finally:137      SetEnvVar(FAIL_FAST_ENV_VAR, None)138  def testGoogletestEnvVarOverridesBazelEnvVar(self):139    """Tests that the Googletest native env var over Bazel testbridge."""140    try:141      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')142      SetEnvVar(FAIL_FAST_ENV_VAR, '1')143      txt, _ = RunAndReturnOutput('HasSimpleTest')144      self.assertIn('1 FAILED TEST', txt)145      self.assertIn('[  SKIPPED ] 3 tests', txt)146    finally:147      SetEnvVar(FAIL_FAST_ENV_VAR, None)148      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)149  def testEventListener(self):150    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)151    self.assertIn('1 FAILED TEST', txt)152    self.assertIn('[  SKIPPED ] 3 tests', txt)153    for expected_count, callback in [(1, 'OnTestSuiteStart'),154                                     (5, 'OnTestStart'),155                                     (5, 'OnTestEnd'),156                                     (5, 'OnTestPartResult'),157                                     (1, 'OnTestSuiteEnd')]:158      self.assertEqual(159          expected_count, txt.count(callback),160          'Expected %d calls to callback %s match count on output: %s ' %161          (expected_count, callback, txt))162    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)163    self.assertIn('3 FAILED TEST', txt)164    self.assertIn('[  SKIPPED ] 1 test', txt)165    for expected_count, callback in [(1, 'OnTestSuiteStart'),166                                     (5, 'OnTestStart'),167                                     (5, 'OnTestEnd'),168                                     (5, 'OnTestPartResult'),169                                     (1, 'OnTestSuiteEnd')]:170      self.assertEqual(171          expected_count, txt.count(callback),172          'Expected %d calls to callback %s match count on output: %s ' %173          (expected_count, callback, txt))174  def assertXmlResultCount(self, result, count, xml):175    self.assertEqual(176        count, xml.count('result="%s"' % result),177        'Expected \'result="%s"\' match count of %s: %s ' %178        (result, count, xml))179  def assertXmlStatusCount(self, status, count, xml):180    self.assertEqual(181        count, xml.count('status="%s"' % status),182        'Expected \'status="%s"\' match count of %s: %s ' %183        (status, count, xml))184  def assertFailFastXmlAndTxtOutput(self,185                                    fail_fast,186                                    test_suite,187                                    passed_count,188                                    failure_count,189                                    skipped_count,190                                    suppressed_count,191                                    run_disabled=False):192    """Assert XML and text output of a test execution."""193    txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)194    if failure_count > 0:195      self.assertIn('%s FAILED TEST' % failure_count, txt)196    if suppressed_count > 0:197      self.assertIn('%s DISABLED TEST' % suppressed_count, txt)198    if skipped_count > 0:199      self.assertIn('[  SKIPPED ] %s tests' % skipped_count, txt)200    self.assertXmlStatusCount('run',201                              passed_count + failure_count + skipped_count, xml)202    self.assertXmlStatusCount('notrun', suppressed_count, xml)203    self.assertXmlResultCount('completed', passed_count + failure_count, xml)204    self.assertXmlResultCount('skipped', skipped_count, xml)205    self.assertXmlResultCount('suppressed', suppressed_count, xml)206  def assertFailFastBehavior(self,207                             test_suite,208                             passed_count,209                             failure_count,210                             skipped_count,211                             suppressed_count,212                             run_disabled=False):213    """Assert --fail_fast via flag."""214    for fail_fast in ('true', '1', 't', True):215      self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,216                                         failure_count, skipped_count,217                                         suppressed_count, run_disabled)218  def assertNotFailFastBehavior(self,219                                test_suite,220                                passed_count,221                                failure_count,222                                skipped_count,223                                suppressed_count,224                                run_disabled=False):225    """Assert --nofail_fast via flag."""226    for fail_fast in ('false', '0', 'f', False):227      self.assertFailFastXmlAndTxtOutput(fail_fast, test_suite, passed_count,228                                         failure_count, skipped_count,229                                         suppressed_count, run_disabled)230  def testFlag_HasFixtureTest(self):231    """Tests the behavior of fail_fast and TEST_F."""232    self.assertFailFastBehavior(233        test_suite='HasFixtureTest',234        passed_count=1,235        failure_count=1,236        skipped_count=3,237        suppressed_count=0)238    self.assertNotFailFastBehavior(239        test_suite='HasFixtureTest',240        passed_count=1,241        failure_count=4,242        skipped_count=0,243        suppressed_count=0)244  def testFlag_HasSimpleTest(self):245    """Tests the behavior of fail_fast and TEST."""246    self.assertFailFastBehavior(247        test_suite='HasSimpleTest',248        passed_count=1,249        failure_count=1,250        skipped_count=3,251        suppressed_count=0)252    self.assertNotFailFastBehavior(253        test_suite='HasSimpleTest',254        passed_count=1,255        failure_count=4,256        skipped_count=0,257        suppressed_count=0)258  def testFlag_HasParametersTest(self):259    """Tests the behavior of fail_fast and TEST_P."""260    self.assertFailFastBehavior(261        test_suite='HasParametersSuite/HasParametersTest',262        passed_count=0,263        failure_count=1,264        skipped_count=3,265        suppressed_count=0)266    self.assertNotFailFastBehavior(267        test_suite='HasParametersSuite/HasParametersTest',268        passed_count=0,269        failure_count=4,270        skipped_count=0,271        suppressed_count=0)272  def testFlag_HasDisabledTest(self):273    """Tests the behavior of fail_fast and Disabled test cases."""274    self.assertFailFastBehavior(275        test_suite='HasDisabledTest',276        passed_count=1,277        failure_count=1,278        skipped_count=2,279        suppressed_count=1,280        run_disabled=False)281    self.assertNotFailFastBehavior(282        test_suite='HasDisabledTest',283        passed_count=1,284        failure_count=3,285        skipped_count=0,286        suppressed_count=1,287        run_disabled=False)288  def testFlag_HasDisabledRunDisabledTest(self):289    """Tests the behavior of fail_fast and Disabled test cases enabled."""290    self.assertFailFastBehavior(291        test_suite='HasDisabledTest',292        passed_count=1,293        failure_count=1,294        skipped_count=3,295        suppressed_count=0,296        run_disabled=True)297    self.assertNotFailFastBehavior(298        test_suite='HasDisabledTest',299        passed_count=1,300        failure_count=4,301        skipped_count=0,302        suppressed_count=0,303        run_disabled=True)304  def testFlag_HasDisabledSuiteTest(self):305    """Tests the behavior of fail_fast and Disabled test suites."""306    self.assertFailFastBehavior(307        test_suite='DISABLED_HasDisabledSuite',308        passed_count=0,309        failure_count=0,310        skipped_count=0,311        suppressed_count=5,312        run_disabled=False)313    self.assertNotFailFastBehavior(314        test_suite='DISABLED_HasDisabledSuite',315        passed_count=0,316        failure_count=0,317        skipped_count=0,318        suppressed_count=5,319        run_disabled=False)320  def testFlag_HasDisabledSuiteRunDisabledTest(self):321    """Tests the behavior of fail_fast and Disabled test suites enabled."""322    self.assertFailFastBehavior(323        test_suite='DISABLED_HasDisabledSuite',324        passed_count=1,325        failure_count=1,326        skipped_count=3,327        suppressed_count=0,328        run_disabled=True)329    self.assertNotFailFastBehavior(330        test_suite='DISABLED_HasDisabledSuite',331        passed_count=1,332        failure_count=4,333        skipped_count=0,334        suppressed_count=0,335        run_disabled=True)336  if SUPPORTS_DEATH_TESTS:337    def testFlag_HasDeathTest(self):338      """Tests the behavior of fail_fast and death tests."""339      self.assertFailFastBehavior(340          test_suite='HasDeathTest',341          passed_count=1,342          failure_count=1,343          skipped_count=3,344          suppressed_count=0)345      self.assertNotFailFastBehavior(346          test_suite='HasDeathTest',347          passed_count=1,348          failure_count=4,349          skipped_count=0,350          suppressed_count=0)351if __name__ == '__main__':...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.