How to use optimise_targets method in hypothesis

Best Python code snippet using hypothesis

test_engine.py

Source:test_engine.py

1# This file is part of Hypothesis, which may be found at2# https://github.com/HypothesisWorks/hypothesis/3#4# Most of this work is copyright (C) 2013-2020 David R. MacIver5# (david@drmaciver.com), but it contains contributions by others. See6# CONTRIBUTING.rst for a full list of people who may hold copyright, and7# consult the git log if you need to determine who owns an individual8# contribution.9#10# This Source Code Form is subject to the terms of the Mozilla Public License,11# v. 2.0. If a copy of the MPL was not distributed with this file, You can12# obtain one at https://mozilla.org/MPL/2.0/.13#14# END HEADER15import re16import time17from random import Random18from unittest.mock import Mock19import pytest20from hypothesis import HealthCheck, Phase, Verbosity, settings21from hypothesis.database import ExampleDatabase, InMemoryExampleDatabase22from hypothesis.errors import FailedHealthCheck, Flaky23from hypothesis.internal.compat import int_from_bytes24from hypothesis.internal.conjecture import engine as engine_module25from hypothesis.internal.conjecture.data import ConjectureData, Overrun, Status26from hypothesis.internal.conjecture.engine import (27    MIN_TEST_CALLS,28    ConjectureRunner,29    ExitReason,30    RunIsComplete,31)32from hypothesis.internal.conjecture.pareto import DominanceRelation, dominance33from hypothesis.internal.conjecture.shrinker import Shrinker, block_program34from hypothesis.internal.conjecture.utils import integer_range35from hypothesis.internal.entropy import deterministic_PRNG36from tests.common.strategies import SLOW, HardToShrink37from tests.common.utils import no_shrink38from tests.conjecture.common import (39    SOME_LABEL,40    TEST_SETTINGS,41    buffer_size_limit,42    run_to_buffer,43    run_to_data,44    shrinking_from,45)46def test_can_index_results():47    @run_to_buffer48    def f(data):49        data.draw_bytes(5)50        data.mark_interesting()51    assert f.index(0) == 052    assert f.count(0) == 553def test_non_cloneable_intervals():54    @run_to_buffer55    def x(data):56        data.draw_bytes(10)57        data.draw_bytes(9)58        data.mark_interesting()59    assert x == bytes(19)60def test_deletable_draws():61    @run_to_buffer62    def x(data):63        while True:64            x = data.draw_bytes(2)65            if x[0] == 255:66                data.mark_interesting()67    assert x == bytes([255, 0])68def zero_dist(random, n):69    return bytes(n)70def test_can_load_data_from_a_corpus():71    key = b"hi there"72    db = ExampleDatabase()73    value = b"=\xc3\xe4l\x81\xe1\xc2H\xc9\xfb\x1a\xb6bM\xa8\x7f"74    db.save(key, value)75    def f(data):76        if data.draw_bytes(len(value)) == value:77            data.mark_interesting()78    runner = ConjectureRunner(f, settings=settings(database=db), database_key=key)79    runner.run()80    (last_data,) = runner.interesting_examples.values()81    assert last_data.buffer == value82    assert len(list(db.fetch(key))) == 183def slow_shrinker():84    strat = HardToShrink()85    def accept(data):86        if data.draw(strat):87            data.mark_interesting()88    return accept89@pytest.mark.parametrize("n", [1, 5])90def test_terminates_shrinks(n, monkeypatch):91    from hypothesis.internal.conjecture import engine92    db = InMemoryExampleDatabase()93    def generate_new_examples(self):94        self.cached_test_function([255] * 1000)95    monkeypatch.setattr(96        ConjectureRunner, "generate_new_examples", generate_new_examples97    )98    monkeypatch.setattr(engine, "MAX_SHRINKS", n)99    runner = ConjectureRunner(100        slow_shrinker(),101        settings=settings(max_examples=5000, database=db),102        random=Random(0),103        database_key=b"key",104    )105    runner.run()106    (last_data,) = runner.interesting_examples.values()107    assert last_data.status == Status.INTERESTING108    assert runner.shrinks == n109    in_db = set(db.data[runner.secondary_key])110    assert len(in_db) == n111def test_detects_flakiness():112    failed_once = [False]113    count = [0]114    def tf(data):115        data.draw_bytes(1)116        count[0] += 1117        if not failed_once[0]:118            failed_once[0] = True119            data.mark_interesting()120    runner = ConjectureRunner(tf)121    runner.run()122    assert runner.exit_reason == ExitReason.flaky123    assert count == [MIN_TEST_CALLS + 1]124def recur(i, data):125    try:126        if i >= 1:127            recur(i - 1, data)128    except RecursionError:129        data.mark_interesting()130def test_recursion_error_is_not_flaky():131    def tf(data):132        i = data.draw_bits(16)133        recur(i, data)134    runner = ConjectureRunner(tf)135    runner.run()136    assert runner.exit_reason == ExitReason.finished137def test_variadic_draw():138    def draw_list(data):139        result = []140        while True:141            data.start_example(SOME_LABEL)142            d = data.draw_bytes(1)[0] & 7143            if d:144                result.append(data.draw_bytes(d))145            data.stop_example()146            if not d:147                break148        return result149    @run_to_buffer150    def b(data):151        if any(all(d) for d in draw_list(data)):152            data.mark_interesting()153    ls = draw_list(ConjectureData.for_buffer(b))154    assert len(ls) == 1155    assert len(ls[0]) == 1156def test_draw_to_overrun():157    @run_to_buffer158    def x(data):159        d = (data.draw_bytes(1)[0] - 8) & 0xFF160        data.draw_bytes(128 * d)161        if d >= 2:162            data.mark_interesting()163    assert x == bytes([10]) + bytes(128 * 2)164def test_can_navigate_to_a_valid_example():165    def f(data):166        i = int_from_bytes(data.draw_bytes(2))167        data.draw_bytes(i)168        data.mark_interesting()169    runner = ConjectureRunner(f, settings=settings(max_examples=5000, database=None))170    with buffer_size_limit(2):171        runner.run()172    assert runner.interesting_examples173def test_stops_after_max_examples_when_reading():174    key = b"key"175    db = ExampleDatabase(":memory:")176    for i in range(10):177        db.save(key, bytes([i]))178    seen = []179    def f(data):180        seen.append(data.draw_bytes(1))181    runner = ConjectureRunner(182        f, settings=settings(max_examples=1, database=db), database_key=key183    )184    runner.run()185    assert len(seen) == 1186def test_stops_after_max_examples_when_generating():187    seen = []188    def f(data):189        seen.append(data.draw_bytes(1))190    runner = ConjectureRunner(f, settings=settings(max_examples=1, database=None))191    runner.run()192    assert len(seen) == 1193@pytest.mark.parametrize("examples", [1, 5, 20, 50])194def test_stops_after_max_examples_when_generating_more_bugs(examples):195    seen = []196    bad = [False, False]197    def f(data):198        seen.append(data.draw_bits(32))199        # Rare, potentially multi-error conditions200        if seen[-1] > 2 ** 31:201            bad[0] = True202            raise ValueError203        bad[1] = True204        raise Exception205    runner = ConjectureRunner(206        f, settings=settings(max_examples=examples, phases=[Phase.generate])207    )208    try:209        runner.run()210    except Exception:211        pass212    # No matter what, whether examples is larger or smalller than MAX_TEST_CALLS,213    # we stop looking at max_examples.  (and re-run each failure for the traceback)214    assert len(seen) <= examples + sum(bad)215def test_interleaving_engines():216    children = []217    @run_to_buffer218    def x(data):219        rnd = Random(data.draw_bytes(1))220        def g(d2):221            d2.draw_bytes(1)222            data.mark_interesting()223        runner = ConjectureRunner(g, random=rnd)224        children.append(runner)225        runner.run()226        if runner.interesting_examples:227            data.mark_interesting()228    assert x == b"\0"229    for c in children:230        assert not c.interesting_examples231def test_phases_can_disable_shrinking():232    seen = set()233    def f(data):234        seen.add(bytes(data.draw_bytes(32)))235        data.mark_interesting()236    runner = ConjectureRunner(237        f, settings=settings(database=None, phases=(Phase.reuse, Phase.generate))238    )239    runner.run()240    assert len(seen) == MIN_TEST_CALLS241def test_reuse_phase_runs_for_max_examples_if_generation_is_disabled():242    with deterministic_PRNG():243        db = InMemoryExampleDatabase()244        for i in range(256):245            db.save(b"key", bytes([i]))246        seen = set()247        def test(data):248            seen.add(data.draw_bits(8))249        ConjectureRunner(250            test,251            settings=settings(max_examples=100, database=db, phases=[Phase.reuse]),252            database_key=b"key",253        ).run()254        assert len(seen) == 100255def test_erratic_draws():256    n = [0]257    with pytest.raises(Flaky):258        @run_to_buffer259        def x(data):260            data.draw_bytes(n[0])261            data.draw_bytes(255 - n[0])262            if n[0] == 255:263                data.mark_interesting()264            else:265                n[0] += 1266def test_no_read_no_shrink():267    count = [0]268    @run_to_buffer269    def x(data):270        count[0] += 1271        data.mark_interesting()272    assert x == b""273    assert count == [1]274def test_one_dead_branch():275    with deterministic_PRNG():276        seen = set()277        @run_to_buffer278        def x(data):279            i = data.draw_bytes(1)[0]280            if i > 0:281                data.mark_invalid()282            i = data.draw_bytes(1)[0]283            if len(seen) < 255:284                seen.add(i)285            elif i not in seen:286                data.mark_interesting()287def test_does_not_save_on_interrupt():288    def interrupts(data):289        raise KeyboardInterrupt()290    db = InMemoryExampleDatabase()291    runner = ConjectureRunner(292        interrupts, settings=settings(database=db), database_key=b"key"293    )294    with pytest.raises(KeyboardInterrupt):295        runner.run()296    assert not db.data297def test_returns_written():298    value = b"\0\1\2\3"299    @run_to_buffer300    def written(data):301        data.write(value)302        data.mark_interesting()303    assert value == written304def fails_health_check(label, **kwargs):305    def accept(f):306        runner = ConjectureRunner(307            f,308            settings=settings(309                max_examples=100, phases=no_shrink, database=None, **kwargs310            ),311        )312        with pytest.raises(FailedHealthCheck) as e:313            runner.run()314        assert e.value.health_check == label315        assert not runner.interesting_examples316    return accept317def test_fails_health_check_for_all_invalid():318    @fails_health_check(HealthCheck.filter_too_much)319    def _(data):320        data.draw_bytes(2)321        data.mark_invalid()322def test_fails_health_check_for_large_base():323    @fails_health_check(HealthCheck.large_base_example)324    def _(data):325        data.draw_bytes(10 ** 6)326def test_fails_health_check_for_large_non_base():327    @fails_health_check(HealthCheck.data_too_large)328    def _(data):329        if data.draw_bits(8):330            data.draw_bytes(10 ** 6)331def test_fails_health_check_for_slow_draws():332    @fails_health_check(HealthCheck.too_slow)333    def _(data):334        data.draw(SLOW)335@pytest.mark.parametrize("n_large", [1, 5, 8, 15])336def test_can_shrink_variable_draws(n_large):337    target = 128 * n_large338    @run_to_data339    def data(data):340        n = data.draw_bits(4)341        b = [data.draw_bits(8) for _ in range(n)]342        if sum(b) >= target:343            data.mark_interesting()344    x = data.buffer345    assert x.count(0) == 0346    assert sum(x[1:]) == target347def test_run_nothing():348    def f(data):349        assert False350    runner = ConjectureRunner(f, settings=settings(phases=()))351    runner.run()352    assert runner.call_count == 0353class Foo:354    def __repr__(self):355        return "stuff"356def test_debug_data(capsys):357    buf = [0, 1, 2]358    def f(data):359        for x in bytes(buf):360            if data.draw_bits(8) != x:361                data.mark_invalid()362            data.start_example(1)363            data.stop_example()364        data.mark_interesting()365    runner = ConjectureRunner(366        f,367        settings=settings(368            max_examples=5000,369            database=None,370            suppress_health_check=HealthCheck.all(),371            verbosity=Verbosity.debug,372        ),373    )374    runner.cached_test_function(buf)375    runner.run()376    out, _ = capsys.readouterr()377    assert re.match("\\d+ bytes \\[.*\\] -> ", out)378    assert "INTERESTING" in out379def test_can_write_bytes_towards_the_end():380    buf = b"\1\2\3"381    def f(data):382        if data.draw_bits(1):383            data.draw_bytes(5)384            data.write(bytes(buf))385            assert bytes(data.buffer[-len(buf) :]) == buf386    with buffer_size_limit(10):387        ConjectureRunner(f).run()388def test_uniqueness_is_preserved_when_writing_at_beginning():389    seen = set()390    def f(data):391        data.write(bytes(1))392        n = data.draw_bits(3)393        assert n not in seen394        seen.add(n)395    runner = ConjectureRunner(f, settings=settings(max_examples=50))396    runner.run()397    assert runner.valid_examples == len(seen)398@pytest.mark.parametrize("skip_target", [False, True])399@pytest.mark.parametrize("initial_attempt", [127, 128])400def test_clears_out_its_database_on_shrinking(401    initial_attempt, skip_target, monkeypatch402):403    def generate_new_examples(self):404        self.cached_test_function(initial_attempt)405    monkeypatch.setattr(406        ConjectureRunner, "generate_new_examples", generate_new_examples407    )408    key = b"key"409    db = InMemoryExampleDatabase()410    def f(data):411        if data.draw_bits(8) >= 127:412            data.mark_interesting()413    runner = ConjectureRunner(414        f,415        settings=settings(database=db, max_examples=256),416        database_key=key,417        random=Random(0),418    )419    for n in range(256):420        if n != 127 or not skip_target:421            db.save(runner.secondary_key, bytes([n]))422    runner.run()423    assert len(runner.interesting_examples) == 1424    for b in db.fetch(runner.secondary_key):425        assert b[0] >= 127426    assert len(list(db.fetch(runner.database_key))) == 1427def test_detects_too_small_block_starts():428    call_count = [0]429    def f(data):430        assert call_count[0] == 0431        call_count[0] += 1432        data.draw_bytes(8)433        data.mark_interesting()434    runner = ConjectureRunner(f, settings=settings(database=None))435    r = runner.cached_test_function(bytes(8))436    assert r.status == Status.INTERESTING437    assert call_count[0] == 1438    r2 = runner.cached_test_function(bytes([255] * 7))439    assert r2.status == Status.OVERRUN440    assert call_count[0] == 1441def test_shrinks_both_interesting_examples(monkeypatch):442    def generate_new_examples(self):443        self.cached_test_function(bytes([1]))444    monkeypatch.setattr(445        ConjectureRunner, "generate_new_examples", generate_new_examples446    )447    def f(data):448        n = data.draw_bits(8)449        data.mark_interesting(n & 1)450    runner = ConjectureRunner(f, database_key=b"key")451    runner.run()452    assert runner.interesting_examples[0].buffer == bytes([0])453    assert runner.interesting_examples[1].buffer == bytes([1])454def test_discarding(monkeypatch):455    monkeypatch.setattr(Shrinker, "shrink", Shrinker.remove_discarded)456    monkeypatch.setattr(457        ConjectureRunner,458        "generate_new_examples",459        lambda runner: runner.cached_test_function(bytes([0, 1] * 10)),460    )461    @run_to_buffer462    def x(data):463        count = 0464        while count < 10:465            data.start_example(SOME_LABEL)466            b = data.draw_bits(1)467            if b:468                count += 1469            data.stop_example(discard=not b)470        data.mark_interesting()471    assert x == bytes(bytes([1]) * 10)472def test_can_remove_discarded_data():473    @shrinking_from(bytes([0] * 10 + [11]))474    def shrinker(data):475        while True:476            data.start_example(SOME_LABEL)477            b = data.draw_bits(8)478            data.stop_example(discard=(b == 0))479            if b == 11:480                break481        data.mark_interesting()482    shrinker.remove_discarded()483    assert list(shrinker.buffer) == [11]484def test_discarding_iterates_to_fixed_point():485    @shrinking_from(bytes(list(range(100, -1, -1))))486    def shrinker(data):487        data.start_example(0)488        data.draw_bits(8)489        data.stop_example(discard=True)490        while data.draw_bits(8):491            pass492        data.mark_interesting()493    shrinker.remove_discarded()494    assert list(shrinker.buffer) == [1, 0]495def test_discarding_is_not_fooled_by_empty_discards():496    @shrinking_from(bytes([1, 1]))497    def shrinker(data):498        data.draw_bits(1)499        data.start_example(0)500        data.stop_example(discard=True)501        data.draw_bits(1)502        data.mark_interesting()503    shrinker.remove_discarded()504    assert shrinker.shrink_target.has_discards505def test_discarding_can_fail(monkeypatch):506    @shrinking_from(bytes([1]))507    def shrinker(data):508        data.start_example(0)509        data.draw_bits(1)510        data.stop_example(discard=True)511        data.mark_interesting()512    shrinker.remove_discarded()513    assert any(e.discarded and e.length > 0 for e in shrinker.shrink_target.examples)514def test_shrinking_from_mostly_zero(monkeypatch):515    monkeypatch.setattr(516        ConjectureRunner,517        "generate_new_examples",518        lambda self: self.cached_test_function(bytes(5) + bytes([2])),519    )520    @run_to_buffer521    def x(data):522        s = [data.draw_bits(8) for _ in range(6)]523        if any(s):524            data.mark_interesting()525    assert x == bytes(5) + bytes([1])526def test_handles_nesting_of_discard_correctly(monkeypatch):527    monkeypatch.setattr(Shrinker, "shrink", Shrinker.remove_discarded)528    monkeypatch.setattr(529        ConjectureRunner,530        "generate_new_examples",531        lambda runner: runner.cached_test_function(bytes([0, 0, 1, 1])),532    )533    @run_to_buffer534    def x(data):535        while True:536            data.start_example(SOME_LABEL)537            succeeded = data.draw_bits(1)538            data.start_example(SOME_LABEL)539            data.draw_bits(1)540            data.stop_example(discard=not succeeded)541            data.stop_example(discard=not succeeded)542            if succeeded:543                data.mark_interesting()544    assert x == bytes([1, 1])545def test_database_clears_secondary_key():546    key = b"key"547    database = InMemoryExampleDatabase()548    def f(data):549        if data.draw_bits(8) == 10:550            data.mark_interesting()551        else:552            data.mark_invalid()553    runner = ConjectureRunner(554        f,555        settings=settings(556            max_examples=1, database=database, suppress_health_check=HealthCheck.all()557        ),558        database_key=key,559    )560    for i in range(10):561        database.save(runner.secondary_key, bytes([i]))562    runner.cached_test_function([10])563    assert runner.interesting_examples564    assert len(set(database.fetch(key))) == 1565    assert len(set(database.fetch(runner.secondary_key))) == 10566    runner.clear_secondary_key()567    assert len(set(database.fetch(key))) == 1568    assert len(set(database.fetch(runner.secondary_key))) == 0569def test_database_uses_values_from_secondary_key():570    key = b"key"571    database = InMemoryExampleDatabase()572    def f(data):573        if data.draw_bits(8) >= 5:574            data.mark_interesting()575        else:576            data.mark_invalid()577    runner = ConjectureRunner(578        f,579        settings=settings(580            max_examples=1, database=database, suppress_health_check=HealthCheck.all()581        ),582        database_key=key,583    )584    for i in range(10):585        database.save(runner.secondary_key, bytes([i]))586    runner.cached_test_function([10])587    assert runner.interesting_examples588    assert len(set(database.fetch(key))) == 1589    assert len(set(database.fetch(runner.secondary_key))) == 10590    runner.clear_secondary_key()591    assert len(set(database.fetch(key))) == 1592    assert set(map(int_from_bytes, database.fetch(runner.secondary_key))) == set(593        range(6, 11)594    )595    (v,) = runner.interesting_examples.values()596    assert list(v.buffer) == [5]597def test_exit_because_max_iterations():598    def f(data):599        data.draw_bits(64)600        data.mark_invalid()601    runner = ConjectureRunner(602        f,603        settings=settings(604            max_examples=1, database=None, suppress_health_check=HealthCheck.all()605        ),606    )607    runner.run()608    assert runner.call_count <= 1000609    assert runner.exit_reason == ExitReason.max_iterations610def test_exit_because_shrink_phase_timeout(monkeypatch):611    val = [0]612    def fast_time():613        val[0] += 1000614        return val[0]615    def f(data):616        if data.draw_bits(64) > 2 ** 33:617            data.mark_interesting()618    monkeypatch.setattr(time, "perf_counter", fast_time)619    runner = ConjectureRunner(f, settings=settings(database=None))620    runner.run()621    assert runner.exit_reason == ExitReason.very_slow_shrinking622    assert runner.statistics["stopped-because"] == "shrinking was very slow"623def test_dependent_block_pairs_can_lower_to_zero():624    @shrinking_from([1, 0, 1])625    def shrinker(data):626        if data.draw_bits(1):627            n = data.draw_bits(16)628        else:629            n = data.draw_bits(8)630        if n == 1:631            data.mark_interesting()632    shrinker.fixate_shrink_passes(["minimize_individual_blocks"])633    assert list(shrinker.shrink_target.buffer) == [0, 1]634def test_handle_size_too_large_during_dependent_lowering():635    @shrinking_from([1, 255, 0])636    def shrinker(data):637        if data.draw_bits(1):638            data.draw_bits(16)639            data.mark_interesting()640        else:641            data.draw_bits(8)642    shrinker.fixate_shrink_passes(["minimize_individual_blocks"])643def test_block_may_grow_during_lexical_shrinking():644    initial = bytes([2, 1, 1])645    @shrinking_from(initial)646    def shrinker(data):647        n = data.draw_bits(8)648        if n == 2:649            data.draw_bits(8)650            data.draw_bits(8)651        else:652            data.draw_bits(16)653        data.mark_interesting()654    shrinker.fixate_shrink_passes(["minimize_individual_blocks"])655    assert list(shrinker.shrink_target.buffer) == [0, 0, 0]656def test_lower_common_block_offset_does_nothing_when_changed_blocks_are_zero():657    @shrinking_from([1, 0, 1, 0])658    def shrinker(data):659        data.draw_bits(1)660        data.draw_bits(1)661        data.draw_bits(1)662        data.draw_bits(1)663        data.mark_interesting()664    shrinker.mark_changed(1)665    shrinker.mark_changed(3)666    shrinker.lower_common_block_offset()667    assert list(shrinker.shrink_target.buffer) == [1, 0, 1, 0]668def test_lower_common_block_offset_ignores_zeros():669    @shrinking_from([2, 2, 0])670    def shrinker(data):671        n = data.draw_bits(8)672        data.draw_bits(8)673        data.draw_bits(8)674        if n > 0:675            data.mark_interesting()676    for i in range(3):677        shrinker.mark_changed(i)678    shrinker.lower_common_block_offset()679    assert list(shrinker.shrink_target.buffer) == [1, 1, 0]680def test_pandas_hack():681    @shrinking_from([2, 1, 1, 7])682    def shrinker(data):683        n = data.draw_bits(8)684        m = data.draw_bits(8)685        if n == 1:686            if m == 7:687                data.mark_interesting()688        data.draw_bits(8)689        if data.draw_bits(8) == 7:690            data.mark_interesting()691    shrinker.fixate_shrink_passes([block_program("-XX")])692    assert list(shrinker.shrink_target.buffer) == [1, 7]693def test_cached_test_function_returns_right_value():694    count = [0]695    def tf(data):696        count[0] += 1697        data.draw_bits(2)698        data.mark_interesting()699    with deterministic_PRNG():700        runner = ConjectureRunner(tf, settings=TEST_SETTINGS)701        for _ in range(2):702            for b in (b"\0", b"\1"):703                d = runner.cached_test_function(b)704                assert d.status == Status.INTERESTING705                assert d.buffer == b706        assert count[0] == 2707def test_cached_test_function_does_not_reinvoke_on_prefix():708    call_count = [0]709    def test_function(data):710        call_count[0] += 1711        data.draw_bits(8)712        data.write(bytes([7]))713        data.draw_bits(8)714    with deterministic_PRNG():715        runner = ConjectureRunner(test_function, settings=TEST_SETTINGS)716        data = runner.cached_test_function(bytes(3))717        assert data.status == Status.VALID718        for n in [2, 1, 0]:719            prefix_data = runner.cached_test_function(bytes(n))720            assert prefix_data is Overrun721        assert call_count[0] == 1722def test_will_evict_entries_from_the_cache(monkeypatch):723    monkeypatch.setattr(engine_module, "CACHE_SIZE", 5)724    count = [0]725    def tf(data):726        data.draw_bytes(1)727        count[0] += 1728    runner = ConjectureRunner(tf, settings=TEST_SETTINGS)729    for _ in range(3):730        for n in range(10):731            runner.cached_test_function([n])732    # Because we exceeded the cache size, our previous733    # calls will have been evicted, so each call to734    # cached_test_function will have to reexecute.735    assert count[0] == 30736def test_branch_ending_in_write():737    seen = set()738    def tf(data):739        count = 0740        while data.draw_bits(1):741            count += 1742        if count > 1:743            data.draw_bits(1, forced=0)744        b = bytes(data.buffer)745        assert b not in seen746        seen.add(b)747    with deterministic_PRNG():748        runner = ConjectureRunner(tf, settings=TEST_SETTINGS)749        for _ in range(100):750            prefix = runner.generate_novel_prefix()751            attempt = prefix + bytes(2)752            data = runner.cached_test_function(attempt)753            assert data.status == Status.VALID754            assert attempt.startswith(data.buffer)755def test_exhaust_space():756    with deterministic_PRNG():757        runner = ConjectureRunner(758            lambda data: data.draw_bits(1), settings=TEST_SETTINGS759        )760        runner.run()761        assert runner.tree.is_exhausted762        assert runner.valid_examples == 2763SMALL_COUNT_SETTINGS = settings(TEST_SETTINGS, max_examples=500)764def test_discards_kill_branches():765    starts = set()766    with deterministic_PRNG():767        def test(data):768            assert runner.call_count <= 256769            while True:770                data.start_example(1)771                b = data.draw_bits(8)772                data.stop_example(b != 0)773                if len(data.buffer) == 1:774                    s = bytes(data.buffer)775                    assert s not in starts776                    starts.add(s)777                if b == 0:778                    break779        runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)780        runner.run()781        assert runner.call_count == 256782@pytest.mark.parametrize("n", range(1, 32))783def test_number_of_examples_in_integer_range_is_bounded(n):784    with deterministic_PRNG():785        def test(data):786            assert runner.call_count <= 2 * n787            integer_range(data, 0, n)788        runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)789        runner.run()790def test_prefix_cannot_exceed_buffer_size(monkeypatch):791    buffer_size = 10792    monkeypatch.setattr(engine_module, "BUFFER_SIZE", buffer_size)793    with deterministic_PRNG():794        def test(data):795            while data.draw_bits(1):796                assert len(data.buffer) <= buffer_size797            assert len(data.buffer) <= buffer_size798        runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)799        runner.run()800        assert runner.valid_examples == buffer_size801def test_does_not_shrink_multiple_bugs_when_told_not_to():802    def test(data):803        m = data.draw_bits(8)804        n = data.draw_bits(8)805        if m > 0:806            data.mark_interesting(1)807        if n > 5:808            data.mark_interesting(2)809    with deterministic_PRNG():810        runner = ConjectureRunner(811            test, settings=settings(TEST_SETTINGS, report_multiple_bugs=False)812        )813        runner.cached_test_function([255, 255])814        runner.shrink_interesting_examples()815        results = {d.buffer for d in runner.interesting_examples.values()}816    assert len(results.intersection({bytes([0, 1]), bytes([1, 0])})) == 1817def test_does_not_keep_generating_when_multiple_bugs():818    def test(data):819        if data.draw_bits(64) > 0:820            data.draw_bits(64)821            data.mark_interesting()822    with deterministic_PRNG():823        runner = ConjectureRunner(824            test,825            settings=settings(826                TEST_SETTINGS, report_multiple_bugs=False, phases=[Phase.generate]827            ),828        )829        runner.run()830    assert runner.call_count == 2831def test_shrink_after_max_examples():832    """If we find a bug, keep looking for more, and then hit the valid-example833    limit, we should still proceed to shrinking.834    """835    max_examples = 100836    fail_at = max_examples - 5837    seen = set()838    bad = set()839    post_failure_calls = [0]840    def test(data):841        if bad:842            post_failure_calls[0] += 1843        value = data.draw_bits(8)844        if value in seen and value not in bad:845            return846        seen.add(value)847        if len(seen) == fail_at:848            bad.add(value)849        if value in bad:850            data.mark_interesting()851    # This shouldn't need to be deterministic, but it makes things much easier852    # to debug if anything goes wrong.853    with deterministic_PRNG():854        runner = ConjectureRunner(855            test,856            settings=settings(857                TEST_SETTINGS,858                max_examples=max_examples,859                phases=[Phase.generate, Phase.shrink],860                report_multiple_bugs=True,861            ),862        )863        runner.shrink_interesting_examples = Mock(name="shrink_interesting_examples")864        runner.run()865    # First, verify our test assumptions: we found a bug, kept running, and866    # then hit max-examples.867    assert runner.interesting_examples868    assert post_failure_calls[0] >= (max_examples - fail_at)869    assert runner.call_count >= max_examples870    assert runner.valid_examples == max_examples871    # Now check that we still performed shrinking, even after hitting the872    # example limit.873    assert runner.shrink_interesting_examples.call_count == 1874    assert runner.exit_reason == ExitReason.finished875def test_shrink_after_max_iterations():876    """If we find a bug, keep looking for more, and then hit the test call877    limit, we should still proceed to shrinking.878    """879    max_examples = 10880    max_iterations = max_examples * 10881    fail_at = max_iterations - 5882    invalid = set()883    bad = set()884    post_failure_calls = [0]885    def test(data):886        if bad:887            post_failure_calls[0] += 1888        value = data.draw_bits(16)889        if value in invalid:890            data.mark_invalid()891        if value in bad or (not bad and len(invalid) == fail_at):892            bad.add(value)893            data.mark_interesting()894        invalid.add(value)895        data.mark_invalid()896    # This shouldn't need to be deterministic, but it makes things much easier897    # to debug if anything goes wrong.898    with deterministic_PRNG():899        runner = ConjectureRunner(900            test,901            settings=settings(902                TEST_SETTINGS,903                max_examples=max_examples,904                phases=[Phase.generate, Phase.shrink],905                report_multiple_bugs=True,906            ),907        )908        runner.shrink_interesting_examples = Mock(name="shrink_interesting_examples")909        runner.run()910    # First, verify our test assumptions: we found a bug, kept running, and911    # then hit the test call limit.912    assert runner.interesting_examples913    assert post_failure_calls[0] >= (max_iterations - fail_at) - 1914    assert runner.call_count >= max_iterations915    assert runner.valid_examples == 0916    # Now check that we still performed shrinking, even after hitting the917    # test call limit.918    assert runner.shrink_interesting_examples.call_count == 1919    assert runner.exit_reason == ExitReason.finished920def test_populates_the_pareto_front():921    with deterministic_PRNG():922        def test(data):923            data.target_observations[""] = data.draw_bits(4)924        runner = ConjectureRunner(925            test,926            settings=settings(927                max_examples=5000,928                database=InMemoryExampleDatabase(),929                suppress_health_check=HealthCheck.all(),930            ),931            database_key=b"stuff",932        )933        runner.run()934        assert len(runner.pareto_front) == 2 ** 4935def test_pareto_front_contains_smallest_valid_when_not_targeting():936    with deterministic_PRNG():937        def test(data):938            data.draw_bits(4)939        runner = ConjectureRunner(940            test,941            settings=settings(942                max_examples=5000,943                database=InMemoryExampleDatabase(),944                suppress_health_check=HealthCheck.all(),945            ),946            database_key=b"stuff",947        )948        runner.run()949        assert len(runner.pareto_front) == 1950def test_pareto_front_contains_different_interesting_reasons():951    with deterministic_PRNG():952        def test(data):953            data.mark_interesting(data.draw_bits(4))954        runner = ConjectureRunner(955            test,956            settings=settings(957                max_examples=5000,958                database=InMemoryExampleDatabase(),959                suppress_health_check=HealthCheck.all(),960            ),961            database_key=b"stuff",962        )963        runner.run()964        assert len(runner.pareto_front) == 2 ** 4965def test_database_contains_only_pareto_front():966    with deterministic_PRNG():967        def test(data):968            data.target_observations["1"] = data.draw_bits(4)969            data.draw_bits(64)970            data.target_observations["2"] = data.draw_bits(8)971        db = InMemoryExampleDatabase()972        runner = ConjectureRunner(973            test,974            settings=settings(975                max_examples=500, database=db, suppress_health_check=HealthCheck.all()976            ),977            database_key=b"stuff",978        )979        runner.run()980        assert len(runner.pareto_front) <= 500981        for v in runner.pareto_front:982            assert v.status >= Status.VALID983        assert len(db.data) == 1984        (values,) = db.data.values()985        values = set(values)986        assert len(values) == len(runner.pareto_front)987        for data in runner.pareto_front:988            assert data.buffer in values989            assert data in runner.pareto_front990        for k in values:991            assert runner.cached_test_function(k) in runner.pareto_front992def test_clears_defunct_pareto_front():993    with deterministic_PRNG():994        def test(data):995            data.draw_bits(8)996            data.draw_bits(8)997        db = InMemoryExampleDatabase()998        runner = ConjectureRunner(999            test,1000            settings=settings(1001                max_examples=10000,1002                database=db,1003                suppress_health_check=HealthCheck.all(),1004                phases=[Phase.reuse],1005            ),1006            database_key=b"stuff",1007        )1008        for i in range(256):1009            db.save(runner.pareto_key, bytes([i, 0]))1010        runner.run()1011        assert len(list(db.fetch(runner.pareto_key))) == 11012def test_replaces_all_dominated():1013    def test(data):1014        data.target_observations["m"] = 3 - data.draw_bits(2)1015        data.target_observations["n"] = 3 - data.draw_bits(2)1016    runner = ConjectureRunner(1017        test,1018        settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),1019        database_key=b"stuff",1020    )1021    d1 = runner.cached_test_function([0, 1]).as_result()1022    d2 = runner.cached_test_function([1, 0]).as_result()1023    assert len(runner.pareto_front) == 21024    assert runner.pareto_front[0] == d11025    assert runner.pareto_front[1] == d21026    d3 = runner.cached_test_function([0, 0]).as_result()1027    assert len(runner.pareto_front) == 11028    assert runner.pareto_front[0] == d31029def test_does_not_duplicate_elements():1030    def test(data):1031        data.target_observations["m"] = data.draw_bits(8)1032    runner = ConjectureRunner(1033        test,1034        settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),1035        database_key=b"stuff",1036    )1037    d1 = runner.cached_test_function([1]).as_result()1038    assert len(runner.pareto_front) == 11039    # This can happen in practice if we e.g. reexecute a test because it has1040    # expired from the cache. It's easier just to test it directly though1041    # rather than simulate the failure mode.1042    is_pareto = runner.pareto_front.add(d1)1043    assert is_pareto1044    assert len(runner.pareto_front) == 11045def test_includes_right_hand_side_targets_in_dominance():1046    def test(data):1047        if data.draw_bits(8):1048            data.target_observations[""] = 101049    runner = ConjectureRunner(1050        test,1051        settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),1052        database_key=b"stuff",1053    )1054    d1 = runner.cached_test_function([0]).as_result()1055    d2 = runner.cached_test_function([1]).as_result()1056    assert dominance(d1, d2) == DominanceRelation.NO_DOMINANCE1057def test_smaller_interesting_dominates_larger_valid():1058    def test(data):1059        if data.draw_bits(8) == 0:1060            data.mark_interesting()1061    runner = ConjectureRunner(1062        test,1063        settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),1064        database_key=b"stuff",1065    )1066    d1 = runner.cached_test_function([0]).as_result()1067    d2 = runner.cached_test_function([1]).as_result()1068    assert dominance(d1, d2) == DominanceRelation.LEFT_DOMINATES1069def test_runs_full_set_of_examples():1070    def test(data):1071        data.draw_bits(64)1072    runner = ConjectureRunner(1073        test,1074        settings=settings(TEST_SETTINGS, database=InMemoryExampleDatabase()),1075        database_key=b"stuff",1076    )1077    runner.run()1078    assert runner.valid_examples == TEST_SETTINGS.max_examples1079def test_runs_optimisation_even_if_not_generating():1080    def test(data):1081        data.target_observations["n"] = data.draw_bits(16)1082    with deterministic_PRNG():1083        runner = ConjectureRunner(1084            test, settings=settings(TEST_SETTINGS, phases=[Phase.target])1085        )1086        runner.cached_test_function(bytes(2))1087        runner.run()1088        assert runner.best_observed_targets["n"] == (2 ** 16) - 11089def test_runs_optimisation_once_when_generating():1090    def test(data):1091        data.target_observations["n"] = data.draw_bits(16)1092    with deterministic_PRNG():1093        runner = ConjectureRunner(1094            test, settings=settings(TEST_SETTINGS, max_examples=100)1095        )1096        runner.optimise_targets = Mock(name="optimise_targets")1097        try:1098            runner.generate_new_examples()1099        except RunIsComplete:1100            pass1101        assert runner.optimise_targets.call_count == 11102def test_does_not_run_optimisation_when_max_examples_is_small():1103    def test(data):1104        data.target_observations["n"] = data.draw_bits(16)1105    with deterministic_PRNG():1106        runner = ConjectureRunner(1107            test, settings=settings(TEST_SETTINGS, max_examples=10)1108        )1109        runner.optimise_targets = Mock(name="optimise_targets")1110        try:1111            runner.generate_new_examples()1112        except RunIsComplete:1113            pass1114        assert runner.optimise_targets.call_count == 01115def test_does_not_cache_extended_prefix():1116    def test(data):1117        data.draw_bits(64)1118    with deterministic_PRNG():1119        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1120        d1 = runner.cached_test_function(b"", extend=8)1121        d2 = runner.cached_test_function(b"", extend=8)1122        assert d1.status == d2.status == Status.VALID1123        assert d1.buffer != d2.buffer1124def test_does_cache_if_extend_is_not_used():1125    calls = [0]1126    def test(data):1127        calls[0] += 11128        data.draw_bits(8)1129    with deterministic_PRNG():1130        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1131        d1 = runner.cached_test_function(b"\0", extend=8)1132        d2 = runner.cached_test_function(b"\0", extend=8)1133        assert d1.status == d2.status == Status.VALID1134        assert d1.buffer == d2.buffer1135        assert calls[0] == 11136def test_does_result_for_reuse():1137    calls = [0]1138    def test(data):1139        calls[0] += 11140        data.draw_bits(8)1141    with deterministic_PRNG():1142        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1143        d1 = runner.cached_test_function(b"\0", extend=8)1144        d2 = runner.cached_test_function(d1.buffer)1145        assert d1.status == d2.status == Status.VALID1146        assert d1.buffer == d2.buffer1147        assert calls[0] == 11148def test_does_not_cache_overrun_if_extending():1149    def test(data):1150        data.draw_bits(64)1151    with deterministic_PRNG():1152        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1153        d1 = runner.cached_test_function(b"", extend=4)1154        d2 = runner.cached_test_function(b"", extend=8)1155        assert d1.status == Status.OVERRUN1156        assert d2.status == Status.VALID1157def test_does_cache_overrun_if_not_extending():1158    def test(data):1159        data.draw_bits(64)1160        data.draw_bits(64)1161    with deterministic_PRNG():1162        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1163        d1 = runner.cached_test_function(bytes(8), extend=0)1164        d2 = runner.cached_test_function(bytes(8), extend=8)1165        assert d1.status == Status.OVERRUN1166        assert d2.status == Status.VALID1167def test_does_not_cache_extended_prefix_if_overrun():1168    def test(data):1169        data.draw_bits(64)1170    with deterministic_PRNG():1171        runner = ConjectureRunner(test, settings=TEST_SETTINGS)1172        d1 = runner.cached_test_function(b"", extend=4)1173        d2 = runner.cached_test_function(b"", extend=8)1174        assert d1.status == Status.OVERRUN1175        assert d2.status == Status.VALID1176def test_can_be_set_to_ignore_limits():1177    def test(data):1178        data.draw_bits(8)1179    with deterministic_PRNG():1180        runner = ConjectureRunner(1181            test, settings=settings(TEST_SETTINGS, max_examples=1), ignore_limits=True1182        )1183        for c in range(256):1184            runner.cached_test_function([c])...

test_optimiser.py

Source:test_optimiser.py

...25            data.target_observations["m"] = data.draw_bits(8)26        runner = ConjectureRunner(test, settings=TEST_SETTINGS)27        runner.cached_test_function([0])28        try:29            runner.optimise_targets()30        except RunIsComplete:31            pass32        assert runner.best_observed_targets["m"] == 25533def test_optimises_multiple_targets():34    with deterministic_PRNG():35        def test(data):36            n = data.draw_bits(8)37            m = data.draw_bits(8)38            if n + m > 256:39                data.mark_invalid()40            data.target_observations["m"] = m41            data.target_observations["n"] = n42            data.target_observations["m + n"] = m + n43        runner = ConjectureRunner(test, settings=TEST_SETTINGS)44        runner.cached_test_function([200, 0])45        runner.cached_test_function([0, 200])46        try:47            runner.optimise_targets()48        except RunIsComplete:49            pass50        assert runner.best_observed_targets["m"] == 25551        assert runner.best_observed_targets["n"] == 25552        assert runner.best_observed_targets["m + n"] == 25653def test_optimises_when_last_element_is_empty():54    with deterministic_PRNG():55        def test(data):56            data.target_observations["n"] = data.draw_bits(8)57            data.start_example(1)58            data.stop_example(1)59        runner = ConjectureRunner(test, settings=TEST_SETTINGS)60        runner.cached_test_function([250])61        try:62            runner.optimise_targets()63        except RunIsComplete:64            pass65        assert runner.best_observed_targets["n"] == 25566def test_can_optimise_last_with_following_empty():67    with deterministic_PRNG():68        def test(data):69            for _ in range(100):70                data.draw_bits(2)71            data.target_observations[""] = data.draw_bits(8)72            data.start_example(1)73            data.stop_example()74        runner = ConjectureRunner(75            test, settings=settings(TEST_SETTINGS, max_examples=100)76        )77        runner.cached_test_function(bytes(101))78        with pytest.raises(RunIsComplete):79            runner.optimise_targets()80        assert runner.best_observed_targets[""] == 25581@pytest.mark.parametrize("lower, upper", [(0, 1000), (13, 100), (1000, 2 ** 16 - 1)])82@pytest.mark.parametrize("score_up", [False, True])83def test_can_find_endpoints_of_a_range(lower, upper, score_up):84    with deterministic_PRNG():85        def test(data):86            n = data.draw_bits(16)87            if n < lower or n > upper:88                data.mark_invalid()89            if not score_up:90                n = -n91            data.target_observations["n"] = n92        runner = ConjectureRunner(93            test, settings=settings(TEST_SETTINGS, max_examples=1000)94        )95        runner.cached_test_function(int_to_bytes((lower + upper) // 2, 2))96        try:97            runner.optimise_targets()98        except RunIsComplete:99            pass100        if score_up:101            assert runner.best_observed_targets["n"] == upper102        else:103            assert runner.best_observed_targets["n"] == -lower104def test_targeting_can_drive_length_very_high():105    with deterministic_PRNG():106        def test(data):107            count = 0108            while data.draw_bits(2) == 3:109                count += 1110            data.target_observations[""] = min(count, 100)111        runner = ConjectureRunner(test, settings=TEST_SETTINGS)112        runner.cached_test_function(bytes(10))113        try:114            runner.optimise_targets()115        except RunIsComplete:116            pass117        assert runner.best_observed_targets[""] == 100118def test_optimiser_when_test_grows_buffer_to_invalid():119    with deterministic_PRNG():120        def test(data):121            m = data.draw_bits(8)122            data.target_observations["m"] = m123            if m > 100:124                data.draw_bits(16)125                data.mark_invalid()126        runner = ConjectureRunner(test, settings=TEST_SETTINGS)127        runner.cached_test_function(bytes(10))128        try:129            runner.optimise_targets()130        except RunIsComplete:131            pass132        assert runner.best_observed_targets["m"] == 100133def test_can_patch_up_examples():134    with deterministic_PRNG():135        def test(data):136            data.start_example(42)137            m = data.draw_bits(6)138            data.target_observations["m"] = m139            for _ in range(m):140                data.draw_bits(1)141            data.stop_example()142            for i in range(4):143                if i != data.draw_bits(8):144                    data.mark_invalid()145        runner = ConjectureRunner(test, settings=TEST_SETTINGS)146        d = runner.cached_test_function([0, 0, 1, 2, 3, 4])147        assert d.status == Status.VALID148        try:149            runner.optimise_targets()150        except RunIsComplete:151            pass152        assert runner.best_observed_targets["m"] == 63153def test_optimiser_when_test_grows_buffer_to_overflow():154    with deterministic_PRNG():155        with buffer_size_limit(2):156            def test(data):157                m = data.draw_bits(8)158                data.target_observations["m"] = m159                if m > 100:160                    data.draw_bits(64)161                    data.mark_invalid()162            runner = ConjectureRunner(test, settings=TEST_SETTINGS)163            runner.cached_test_function(bytes(10))164            try:165                runner.optimise_targets()166            except RunIsComplete:167                pass...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.