Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use test_config_set method in autotest

Best Python code snippet using autotest_python

test_expectations_editor.py

Source:test_expectations_editor.py

1#!/usr/bin/env python2# Copyright (C) 2010 Google Inc. All rights reserved.3#4# Redistribution and use in source and binary forms, with or without5# modification, are permitted provided that the following conditions are6# met:7#8#     * Redistributions of source code must retain the above copyright9# notice, this list of conditions and the following disclaimer.10#     * Redistributions in binary form must reproduce the above11# copyright notice, this list of conditions and the following disclaimer12# in the documentation and/or other materials provided with the13# distribution.14#     * Neither the name of Google Inc. nor the names of its15# contributors may be used to endorse or promote products derived from16# this software without specific prior written permission.17#18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29"""A helper class for reading in and dealing with tests expectations30for layout tests.31"""32import itertools33import json34import logging35import re36from webkitpy.layout_tests.models.test_configuration import TestConfiguration, TestConfigurationConverter37from webkitpy.layout_tests.models import test_expectations38_log = logging.getLogger(__name__)39class BugManager(object):40    """A simple interface for managing bugs from TestExpectationsEditor."""41    def close_bug(self, bug_ids, reference_bug_ids=None):42        raise NotImplementedError("BugManager.close_bug")43    def create_bug(self):44        """Should return a newly created bug id in the form of r"BUG[^\d].*"."""45        raise NotImplementedError("BugManager.create_bug")46class TestExpectationsEditor(object):47    """48    The editor assumes that the expectation data is error-free.49    """50    def __init__(self, expectation_lines, bug_manager):51        self._bug_manager = bug_manager52        self._expectation_lines = expectation_lines53        self._tests_with_directory_paths = set()54        # FIXME: Unify this with TestExpectationsModel.55        self._test_to_expectation_lines = {}56        for expectation_line in expectation_lines:57            for test in expectation_line.matching_tests:58                if test == expectation_line.path:59                    self._test_to_expectation_lines.setdefault(test, []).append(expectation_line)60                else:61                    self._tests_with_directory_paths.add(test)62    def remove_expectation(self, test, test_config_set, remove_flakes=False):63        """Removes existing expectations for {test} in the of test configurations {test_config_set}.64        If the test is flaky, the expectation is not removed, unless remove_flakes is True.65        In this context, removing expectations does not imply that the test is passing -- we are merely removing66        any information about this test from the expectations.67        We do not remove the actual expectation lines here. Instead, we adjust TestExpectationLine.matching_configurations.68        The serializer will figure out what to do:69        * An empty matching_configurations set means that the this line matches nothing and will serialize as None.70        * A matching_configurations set that can't be expressed as one line will be serialized as multiple lines.71        Also, we do only adjust matching_configurations for lines that match tests exactly, because expectation lines with72        better path matches are valid and always win.73        For example, the expectation with the path "fast/events/shadow/" will74        be ignored when removing expectations for the test "fast/event/shadow/awesome-crash.html", since we can just75        add a new expectation line for "fast/event/shadow/awesome-crash.html" to influence expected results.76        """77        expectation_lines = self._test_to_expectation_lines.get(test, [])78        for expectation_line in expectation_lines:79            if (not expectation_line.is_flaky() or remove_flakes) and expectation_line.matching_configurations & test_config_set:80                expectation_line.matching_configurations = expectation_line.matching_configurations - test_config_set81                if not expectation_line.matching_configurations:82                    self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers)83                return84    def update_expectation(self, test, test_config_set, expectation_set, parsed_bug_modifiers=None):85        """Updates expectations for {test} in the set of test configuration {test_config_set} to the values of {expectation_set}.86        If {parsed_bug_modifiers} is supplied, it is used for updated expectations. Otherwise, a new bug is created.87        Here, we treat updating expectations to PASS as special: if possible, the corresponding lines are completely removed.88        """89        # FIXME: Allow specifying modifiers (SLOW, SKIP, WONTFIX).90        updated_expectations = []91        expectation_lines = self._test_to_expectation_lines.get(test, [])92        remaining_configurations = test_config_set.copy()93        bug_ids = self._get_valid_bug_ids(parsed_bug_modifiers)94        new_expectation_line_insertion_point = len(self._expectation_lines)95        remove_expectations = expectation_set == set([test_expectations.PASS]) and test not in self._tests_with_directory_paths96        for expectation_line in expectation_lines:97            if expectation_line.matching_configurations == remaining_configurations:98                # Tweak expectations on existing line.99                if expectation_line.parsed_expectations == expectation_set:100                    return updated_expectations101                self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)102                updated_expectations.append(expectation_line)103                if remove_expectations:104                    expectation_line.matching_configurations = set()105                else:106                    expectation_line.parsed_expectations = expectation_set107                    expectation_line.parsed_bug_modifiers = bug_ids108                return updated_expectations109            elif expectation_line.matching_configurations >= remaining_configurations:110                # 1) Split up into two expectation lines:111                # * one with old expectations (existing expectation_line)112                # * one with new expectations (new expectation_line)113                # 2) Finish looking, since there will be no more remaining configs to test for.114                expectation_line.matching_configurations -= remaining_configurations115                updated_expectations.append(expectation_line)116                new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1117                break118            elif expectation_line.matching_configurations <= remaining_configurations:119                # Remove existing expectation line.120                self._bug_manager.close_bug(expectation_line.parsed_bug_modifiers, bug_ids)121                expectation_line.matching_configurations = set()122                updated_expectations.append(expectation_line)123            else:124                intersection = expectation_line.matching_configurations & remaining_configurations125                if intersection:126                    expectation_line.matching_configurations -= intersection127                    updated_expectations.append(expectation_line)128            new_expectation_line_insertion_point = self._expectation_lines.index(expectation_line) + 1129        if not remove_expectations:130            new_expectation_line = self._create_new_line(test, bug_ids, remaining_configurations, expectation_set)131            updated_expectations.append(new_expectation_line)132            self._expectation_lines.insert(new_expectation_line_insertion_point, new_expectation_line)133        return updated_expectations134    def _get_valid_bug_ids(self, suggested_bug_ids):135        # FIXME: Flesh out creating a bug properly (title, etc.)136        return suggested_bug_ids or [self._bug_manager.create_bug()]137    def _create_new_line(self, name, bug_ids, config_set, expectation_set):138        new_line = test_expectations.TestExpectationLine()139        new_line.name = name140        new_line.parsed_bug_modifiers = bug_ids141        new_line.matching_configurations = config_set142        new_line.parsed_expectations = expectation_set143        # Ensure index integrity for multiple operations.144        self._test_to_expectation_lines.setdefault(name, []).append(new_line)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.