Best Python code snippet using playwright-python
ebb.py
Source:ebb.py  
...73        ''' Sets a property on the active node '''74        assert Scope._top is not None75        Scope._top.properties[name] = value76    @staticmethod77    def set_checked(name, value, expected_type):78        ''' Sets a property on the active node if it's not None, and checks it's79            type '''80        if value is None:81            return82        if expected_type is not None:83            assert isinstance(value, expected_type)84        Scope.set(name, value)85    @staticmethod86    def append(name, *values):87        ''' Appends values to a list property with key name on this node '''88        assert isinstance(name, basestring)89        assert Scope._top is not None90        properties = Scope._top.properties91        if name not in properties:92            properties[name] = []93        assert isinstance(properties[name], list)94        properties[name].extend(values)95    @staticmethod96    def update(name, key, value):97        ''' Adds a {key : value} entry on the dictionary named name on the top98            node '''99        assert isinstance(name, basestring)100        assert Scope._top is not None101        properties = Scope._top.properties102        if name not in properties:103            properties[name] = {}104        assert isinstance(properties[name], dict)105        properties[name][key] = value106    @staticmethod107    @contextlib.contextmanager108    def push(func, *args, **kwargs):109        ''' Pushes a scope and calls a function before yielding '''110        with Scope() as scope:111            func(*args, **kwargs)112            yield scope113    def get(self, name, default=None, public_only=False):114        ''' Search for a property up the tree and returns the first occurence115        '''116        assert isinstance(name, basestring)117        if name not in self.properties:118            value = None119        else:120            value = self.properties[name]121        if value is not None:122            if not isinstance(value, list) and not isinstance(value, dict):123                return value124        for scope in self._get_related_scopes(public_only):125            scope_value = scope.get(name, public_only=True)126            if isinstance(scope_value, list):127                if value is not None:128                    assert isinstance(value, list)129                    scope_value.extend(value)130            elif isinstance(scope_value, dict):131                if value is not None:132                    assert isinstance(value, dict)133                    scope_value.update(value)134            if scope_value is not None:135                value = scope_value136        if isinstance(value, list):137            return value[:]138        elif isinstance(value, dict):139            return value.copy()140        return value if value is not None else default141    def get_interpolated(self, name, default=None):142        ''' Search for a property up the tree, and returns it's value143            interpolated with this node and all of it's parents property values144        '''145        assert isinstance(name, basestring)146        result = self.get(name, default)147        return self.interpolate(result)148    def interpolate(self, value):149        ''' Interpolates value with all properties from current node and it's150            parents151        '''152        if hasattr(value, '__call__'):153            return value(self)154        elif isinstance(value, basestring):155            format_args = self.get_interpolation_values()156            try:157                return value.format(**format_args)158            except KeyError as error:159                raise KeyError('Key %s not found while interpolating %s on %s' % (error, value, self))160        elif isinstance(value, list):161            return [self.interpolate(it) for it in value]162        elif isinstance(value, dict):163            result = {}164            for key, it in value.iteritems():165                result[key] = self.interpolate(it)166            return result167        return value168    def get_rendered(self, name, default=None):169        ''' Search for a property up the tree, and returns it's value170            interpolated with this node and all of it's parents property values171        '''172        assert isinstance(name, basestring)173        result = self.get(name, default)174        return self.render(result)175    def render(self, value):176        ''' Interpolates value with all properties from current node and it's177            parents178        '''179        if isinstance(value, basestring):180            return _Renderer(value, self)181        elif isinstance(value, list):182            return [self.render(it) for it in value]183        elif isinstance(value, dict):184            result = {}185            for key, it in value.iteritems():186                result[key] = self.render(it)187            return result188        return value189    def get_interpolation_values(self, public_only=False):190        ''' Parses the tree bottom-up, getting string property values '''191        result = {}192        for scope in self._get_related_scopes(public_only):193            scope_values = scope.get_interpolation_values(True)194            result.update(scope_values)195        for key, value in self.properties.iteritems():196            if isinstance(value, basestring):197                result[key] = value198        return result199    def get_parent_of_type(self, parent_type):200        ''' Find closest parent of specified node type '''201        if self._parent is None:202            return None203        if isinstance(self._parent, parent_type):204            return self._parent205        return self._parent.get_parent_of_type(parent_type)206    def build(self, config):207        ''' Builds this node '''208        for child in self.children:209            child.build(config)210        self._build(config)211    def _get_related_scopes(self, public_only):212        if self._parent is not None:213            yield self._parent214        if not public_only:215            for it in self.children:216                if isinstance(it, Private):217                    yield it218    def get_properties_names(self, prefix, public_only=False):219        ''' Returs all properties defined on this node and parent starting220            with given prefix '''221        result = set()222        for scope in self._get_related_scopes(public_only):223            result = result | scope.get_properties_names(prefix, True)224        for key, _ in self.properties.iteritems():225            if key.startswith(prefix + '_'):226                result.add(key)227        return result228    @abc.abstractmethod229    def _build(self, config):230        pass231    def _get_prefixed_properties(self, prefixes, rendered=None, raw=None):232        result = {}233        rendered = set([] if rendered is None else rendered)234        raw = set(() if raw is None else raw)235        if not isinstance(prefixes, tuple):236            assert isinstance(prefixes, str)237            prefixes = (prefixes,)238        props = {}239        for prefix in prefixes:240            interpolated_it = self.get_properties_names(prefix)241            raw_it = interpolated_it & raw242            rendered_it = interpolated_it & rendered243            interpolated_it = interpolated_it - raw_it - rendered_it244            props[prefix] = (interpolated_it, raw_it, rendered_it)245        for prefix, (interpolated_it, raw_it, rendered_it) in props.iteritems():246            for key in interpolated_it:247                key_without_prefix = key[len(prefix) + 1:]248                result[key_without_prefix] = self.get_interpolated(key)249            for key in raw_it:250                key_without_prefix = key[len(prefix) + 1:]251                result[key_without_prefix] = self.get(key)252            for key in rendered_it:253                key_without_prefix = key[len(prefix) + 1:]254                result[key_without_prefix] = self.get_rendered(key)255            # result[name] = self.properties[key]256        return result257    def _build_class(self,258                     buildbot_class,259                     prefixes,260                     positional=None,261                     raw=None,262                     rendered=None,263                     additional=None):264        kwargs = self._get_prefixed_properties(prefixes,265                                               rendered=rendered,266                                               raw=raw)267        if additional is not None:268            kwargs.update(additional)269        args = []270        if positional is not None:271            for name in positional:272                assert name in kwargs, '%s argument missing' % name273                args.append(kwargs[name])274                del kwargs[name]275        return buildbot_class(*args, **kwargs)276class Private(Scope):277    ''' Defines not inherited values on the parent scope '''278    def __init__(self):279        super(Private, self).__init__()280    def _get_related_scopes(self, public_only):281        return []282    def _build(self, config):283        pass284class Config(Scope):285    ''' Root config node '''286    def __init__(self):287        super(Config, self).__init__()288        self.buildbot_config = {}289        self._parsers = {}290        self._schedulers = {}291        self._slaves = []292        self._triggerables = {}293        self._locks = {}294        self._builders_scopes = {}295        self.buildbot_config['builders'] = []296        self.buildbot_config['schedulers'] = []297        self.buildbot_config['slaves'] = []298        self.buildbot_config['status'] = []299        self.buildbot_config['change_source'] = []300        self.buildbot_config['prioritizeBuilders'] = self._prioritize_builders301        self.slave_list_selector = None302        self.next_slave_selector = None303    @staticmethod304    def db(url, poll_interval=None):305        ''' Configures db parameters '''306        Scope.set_checked('db_db_url', url, basestring)307        Scope.set_checked('db_poll_interval', poll_interval, int)308    @staticmethod309    def site(title, title_url, buildbot_url):310        ''' Configures site parameters '''311        Scope.set_checked('base_title', title, basestring)312        Scope.set_checked('base_titleURL', title_url, basestring)313        Scope.set_checked('base_buildbotURL', buildbot_url, basestring)314    @staticmethod315    def logging(compression_limit=None,316                compression_method=None,317                max_size=None,318                max_tail_size=None):319        ''' Configures logging parameters '''320        Scope.set_checked('base_logCompressionLimit', compression_limit, int)321        Scope.set_checked('base_logCompressionMethod',322                          compression_method,323                          basestring)324        Scope.set_checked('base_logMaxSize', max_size, int)325        Scope.set_checked('base_logMaxTailSize', max_tail_size, int)326    @staticmethod327    def horizons(change_horizon=None,328                 build_horizon=None,329                 event_horizon=None,330                 log_horizon=None):331        ''' Configures horizon parameters '''332        Scope.set_checked('base_changeHorizon', change_horizon, int)333        Scope.set_checked('base_buildHorizon', build_horizon, int)334        Scope.set_checked('base_eventHorizon', event_horizon, int)335        Scope.set_checked('base_logHorizon', log_horizon, int)336    @staticmethod337    def cache(changes=None,338              builds=None,339              chdicts=None,340              build_requests=None,341              source_stamps=None,342              ssdicts=None,343              objectids=None,344              usdicts=None):345        ''' Configures horizon parameters '''346        Scope.set_checked('cache_Changes', changes, int)347        Scope.set_checked('cache_Builds', builds, int)348        Scope.set_checked('cache_chdicts', chdicts, int)349        Scope.set_checked('cache_BuildRequests', build_requests, int)350        Scope.set_checked('cache_SourceStamps', source_stamps, int)351        Scope.set_checked('cache_ssdicts', ssdicts, int)352        Scope.set_checked('cache_objectids', objectids, int)353        Scope.set_checked('cache_usdicts', usdicts, int)354    @staticmethod355    def set_protocol(protocol, port):356        ''' Sets given protocol to given port '''357        Scope.update('protocols_%s' % protocol, 'port', port)358    @staticmethod359    def web_status(port, user, password):360        ''' Sets web status configuration '''361        Scope.set_checked('web_status_port', port, int)362        Scope.set_checked('web_status_user', user, str)363        Scope.set_checked('web_status_password', password, str)364    @staticmethod365    def add_renderer_handlers(*handlers):366        ''' Add rendering handlers that can udpate rendering arguments at build367            time '''368        Scope.append('config_renderer_handlers', *handlers)369    def get_builder(self, name):370        ''' Returns a declared Builder '''371        return self._builders_scopes[name]372    def get_slave(self, name):373        ''' Returns a declared slave '''374        for slave in self._slaves:375            if slave.get_interpolated('slave_name') == name:376                return slave377        return None378    def build_config(self):379        ''' Builds the buildbot config '''380        self.build(self)381        return self.buildbot_config382    def add_slave(self, slave):383        ''' Adds a slave for later tag filtering '''384        self._slaves.append(slave)385    def add_builder(self, builder, scope):386        ''' Adds a builder to this config '''387        self._builders_scopes[builder.name] = scope388        self.buildbot_config['builders'].append(builder)389    def get_slave_list(self, *tags):390        ''' Returns declared slaves matching *all* given tags391            Tags can be excluded if they start with â!â '''392        # TODO: allow the tag1|tag2 syntax for e.g. 'linux|win64'393        result = []394        wanted_tagset, unwanted_tagset = set(), set()395        for tag in tags:396            if tag[:1] != '!':397                wanted_tagset.add(tag)398            else:399                unwanted_tagset.add(tag[1:])400        for slave in self._slaves:401            slave_tagset = set()402            for tag in slave.get_interpolated('_slave_tags', []):403                slave_tagset.add(tag)404            if len(wanted_tagset - slave_tagset) == 0 and \405               len(unwanted_tagset.intersection(slave_tagset)) == 0:406                slave_name = slave.get_interpolated('slave_name')407                result.append(slave_name)408        if len(result) == 0:409            print 'Error : no slave found with tags %s and without tags %s' \410                  % (wanted_tagset, unwanted_tagset)411            for slave in self._slaves:412                slave_tagset = set()413                for tag in slave.get_interpolated('_slave_tags', []):414                    slave_tagset.add(tag)415                args = (slave.get_interpolated('slave_name'),416                        wanted_tagset - slave_tagset)417                print 'Slave %s is missing tags %s' % args418        return result419    def _build(self, config):420        assert config == self421        conf_dict = config.buildbot_config422        # Db config423        conf_dict['db'] = self._get_prefixed_properties('db')424        conf_dict['caches'] = self._get_prefixed_properties('cache')425        conf_dict['protocols'] = self._get_prefixed_properties('protocols')426        conf_dict.update(self._get_prefixed_properties('base'))427        self._add_web_status()428    def _add_web_status(self):429        http_port = self.get('web_status_port')430        if http_port is None:431            return432        users = [(self.get_interpolated('web_status_user'),433                  self.get_interpolated('web_status_password'))]434        auth = buildbot.status.web.auth.BasicAuth(users)435        authz = buildbot.status.web.authz.Authz(auth=auth,436                                                view=True,437                                                gracefulShutdown='auth',438                                                forceBuild='auth',439                                                forceAllBuilds='auth',440                                                pingBuilder='auth',441                                                stopBuild='auth',442                                                stopAllBuilds='auth',443                                                cancelPendingBuild='auth',444                                                showUsersPage='auth',445                                                cleanShutdown='auth')446        web_status = buildbot.status.html.WebStatus(http_port=http_port,447                                                    authz=authz)448        self.buildbot_config['status'].append(web_status)449    def _prioritize_builders(self, _, builders):450        def _get_priority(builder):451            try:452                return self._builders_scopes[builder.name].get('_builder_priority', 0)453            except KeyError:454                return 99999455        builders.sort(key=_get_priority, reverse=True)456        return builders457class Slave(Scope):458    ''' Creates a new buildbot slave '''459    def __init__(self, name):460        super(Slave, self).__init__()461        self.properties['slave_name'] = name462        config = self.get_parent_of_type(Config)463        assert config is not None464        config.add_slave(self)465    @staticmethod466    def config(password=None,467               max_builds=None,468               keepalive_interval=300,469               missing_timeout=None):470        ''' Sets some buildbot slaves settings for current scope '''471        Scope.set_checked('slave_password', password, basestring)472        Scope.set_checked('slave_max_builds', max_builds, int)473        Scope.set_checked('slave_keepalive_interval', keepalive_interval, int)474        Scope.set_checked('slave_missing_timeout', missing_timeout, int)475    @staticmethod476    def add_property(key, value):477        ''' Adds a build property on this slave '''478        Scope.update('slave_properties', key, value)479    @staticmethod480    def add_notified_on_missing(*emails):481        ''' Adds emails to notify when slaves in scope are missing '''482        Scope.append('slave_notify_on_missing', *emails)483    @staticmethod484    def add_tags(*tags):485        ''' Adds specified tags to slaves in scope '''486        Scope.append('_slave_tags', *tags)487    def _build(self, config):488        slave = self._build_class(buildbot.buildslave.BuildSlave, 'slave',489                                  ['name', 'password'])490        config.buildbot_config['slaves'].append(slave)491class Builder(Scope):492    ''' Builder wrapper '''493    def __init__(self, name, category=None, description=None):494        super(Builder, self).__init__()495        self._accept_regex = None496        self._reject_regex = None497        self._factory = buildbot.process.factory.BuildFactory()498        self._nightly = None499        self.properties['builder_name'] = name500        if category is not None:501            self.properties['builder_category'] = category502        if description is not None:503            self.properties['builder_description'] = description504    def add_step(self, step):505        ''' Adds a step to this builder '''506        self._factory.addStep(step)507    def trigger_on_change(self, accept_regex='.*', reject_regex=None):508        ''' Triggers this build on change from source control '''509        self._accept_regex = accept_regex510        self._reject_regex = reject_regex511    def trigger_nightly(self,512                        minute=None,513                        hour=None,514                        day_of_month=None,515                        month=None,516                        day_of_week=None):517        ''' Triggers this build nightly '''518        self._nightly = {'minute' : minute,519                         'hour' : hour,520                         'dayOfMonth': day_of_month,521                         'month' : month,522                         'dayOfWeek' : day_of_week}523    @staticmethod524    def config(name=None,525               category=None,526               build_dir=None,527               slave_build_dir=None,528               next_build=None,529               can_start_build=None,530               merge_requests=None,531               forcable=None,532               only_important=None,533               tree_stable_timer=None,534               file_is_important=None,535               project=None,536               priority=None):537        ''' Sets builder config values '''538        Scope.set_checked('builder_name', name, basestring)539        Scope.set_checked('builder_category', category, basestring)540        Scope.set_checked('builder_builddir', build_dir, None)541        Scope.set_checked('builder_slavebuilddir', slave_build_dir, basestring)542        Scope.set_checked('builder_nextBuild', next_build, None)543        Scope.set_checked('builder_canStartBuild', can_start_build, None)544        Scope.set_checked('builder_mergeRequests', merge_requests, None)545        Scope.set_checked('scheduler_onlyImportant', only_important, bool)546        Scope.set_checked('branch_scheduler_treeStableTimer', tree_stable_timer, int)547        Scope.set_checked('scheduler_fileIsImportant', file_is_important, None)548        Scope.set_checked('change_filter_project', project, basestring)549        Scope.set_checked('_builder_priority', priority, int)550    @staticmethod551    def mail_config(from_address=None,552                    send_to_interested_users=None,553                    subject=None,554                    mode=None,555                    add_logs=None,556                    relay_host=None,557                    smpt_port=None,558                    use_tls=None,559                    smtp_user=None,560                    smtp_password=None,561                    lookup=None,562                    message_formatter=None,563                    template_directory=None,564                    template=None,565                    body_type=None):566        ''' Sets mail related settings '''567        Scope.set_checked('mail_fromaddr', from_address, str)568        Scope.set_checked('mail_sendToInterestedUsers',569                          send_to_interested_users, bool)570        Scope.set_checked('mail_subject', subject, str)571        Scope.set_checked('mail_mode', mode, None)572        Scope.set_checked('mail_addLogs', add_logs, None)573        Scope.set_checked('mail_relayhost', relay_host, str)574        Scope.set_checked('mail_smtpPort', smpt_port, int)575        Scope.set_checked('mail_useTls', use_tls, bool)576        Scope.set_checked('mail_smtpUser', smtp_user, str)577        Scope.set_checked('mail_smtpPassword', smtp_password, str)578        Scope.set_checked('mail_lookup', lookup, None)579        Scope.set_checked('_mail_message_formatter', message_formatter, None)580        Scope.set_checked('_mail_template_directory', template_directory, None)581        Scope.set_checked('_mail_template', template, None)582        Scope.set_checked('_mail_body_type', body_type, None)583    @staticmethod584    def add_extra_recipients(*emails):585        ''' Adds extra recipients to users '''586        Scope.append('mail_extraRecipients', *emails)587    @staticmethod588    def add_slave_tags(*tags):589        ''' Adds builder tags to current scope '''590        Scope.append('_builder_slave_tags', *tags)591    @staticmethod592    def add_env_variable(name, value):593        ''' Adds an envrionment variable to builders in scope '''594        Scope.update('builder_env', name, value)595    @staticmethod596    def add_tags(*tags):597        ''' Adds a tag to a builder '''598        Scope.append('builder_tags', *tags)599    @staticmethod600    def add_property(name, value):601        ''' Adds a property to builders in scope '''602        Scope.update('builder_properties', name, value)603    def _build(self, config):604        if config.slave_list_selector:605            slavenames = config.slave_list_selector(self)606        else:607            slave_tags = self.get_interpolated('_builder_slave_tags', [])608            slavenames = config.get_slave_list(*slave_tags)609        # TODO locks = get_locks('job', config, scope)610        args = {611            'slavenames': slavenames,612            'nextSlave': config.next_slave_selector,613            'factory': self._factory,614        }615        builder = self._build_class(buildbot.config.BuilderConfig, 'builder', additional=args)616        config.add_builder(builder, self)617        self._add_single_branch_scheduler(config)618        self._add_nightly_scheduler(config)619        self._add_mail_status(config)620        parent_trigger = self.get_parent_of_type(Trigger)621        if parent_trigger is not None:622            parent_trigger.add_builder(self.get_interpolated('builder_name'))623    def _add_single_branch_scheduler(self, config):624        if self._accept_regex is None:625            return626        builder_name = self.get_interpolated('builder_name')627        project_name = self.get_interpolated('project_name')628        args = {629            'filter_fn': _ChangeFilter(builder_name, project_name,630                                       self.interpolate(self._accept_regex),631                                       self.interpolate(self._reject_regex))632        }633        change_filter = self._build_class(buildbot.changes.filter.ChangeFilter,634                                          'change_filter',635                                          additional=args)636        args = {'name' : '%s single branch scheduler' % builder_name,637                'builderNames' : [builder_name],638                'change_filter' : change_filter,639                'reason' : 'A CL Triggered this build'}640        scheduler_class = buildbot.schedulers.basic.SingleBranchScheduler641        scheduler = self._build_class(scheduler_class,642                                      ('scheduler', 'branch_scheduler'),643                                      additional=args)644        config.buildbot_config['schedulers'].append(scheduler)645    def _add_nightly_scheduler(self, config):646        if self._nightly is None:647            return648        builder_name = self.get_interpolated('builder_name')649        args = {'name' : '%s nightly scheduler' % builder_name,650                'builderNames' : [builder_name],651                'branch' : None} #We don't use branches the way buildbot expects it652        for key, value in self._nightly.iteritems():653            if value is not None:654                args[key] = value655        scheduler_class = buildbot.schedulers.timed.Nightly656        scheduler = self._build_class(scheduler_class,657                                      'scheduler',658                                      additional=args)659        config.buildbot_config['schedulers'].append(scheduler)660    def _add_mail_status(self, config):661        extra_recipients = self.get_interpolated('mail_extraRecipients')662        send_mail = self.get_interpolated('mail_sendToInterestedUsers')663        if extra_recipients or send_mail:664            formatter = self.get('_mail_message_formatter')665            if formatter is None:666                formatter = _HtmlMailFormatter(self)667            args = {'messageFormatter': formatter,668                    'builders': [self.get_interpolated('builder_name')]}669            mail_status = self._build_class(buildbot.status.mail.MailNotifier,670                                            'mail',671                                            additional=args)672            config.buildbot_config['status'].append(mail_status)673class Repository(Scope):674    ''' Change source base scope '''675    def __init__(self, name, is_polling_enabled):676        super(Repository, self).__init__()677        self.name = name678        self.is_polling_enabled = is_polling_enabled679        Scope.update('source_control_repositories', name, self)680    @staticmethod681    def config(poll_interval=None,682               poll_at_launch=None,683               hitsmax=None):684        ''' Common change source parameters '''685        Scope.set_checked('change_source_pollInterval', poll_interval, int)686        Scope.set_checked('change_source_pollAtLaunch', poll_at_launch, bool)687        Scope.set_checked('change_source_hitsmax', hitsmax, int)688    @abc.abstractmethod689    def get_sync_step(self, config, step_args):690        ''' Returns a step to sync this repository '''691    @abc.abstractmethod692    def _build_change_sources(self, config, args):693        ''' Creates a ChangeSource for this repository '''694    def _build(self, config):695        if not self.is_polling_enabled:696            return697        args = self._get_prefixed_properties('change_source')698        for change_source in self._build_change_sources(config, args):699            # XXX: this attribute works around a bug in buildbot that would700            # cause it to only trigger rebuilds for the first project using701            # that source702            change_source.compare_attrs.append('project')703            config.buildbot_config['change_source'].append(change_source)704class P4StreamSource(buildbot.changes.p4poller.P4Source):705    def __init__(self, **args):706        self._stream = None707        super(P4StreamSource, self).__init__(**args)708    @defer.inlineCallbacks709    #pylint: disable=invalid-name,missing-docstring710    def _get_process_output(self, args):711        base_get_process_output = super(P4StreamSource, self)._get_process_output712        # If action is not 'p4 changes', use the original function713        if 'changes' not in args:714            tmp = yield base_get_process_output(args)715            defer.returnValue(tmp)716        # Last argument is the location we're polling717        location, suffix = args[-1], ""718        if '...' in location:719            n = location.index('...')720            location, suffix = location[:n], location[n:]721        if self._stream:722            location = self._stream723        client = re.sub('[^a-zA-Z0-9]+', '-', 'poll-' + location).lower()724        # All arguments before changes are P4 options725        argc = args.index('changes')726        baseargs = args[:argc]727        # Check whether the location is a stream; otherwise, bail out728        tmp = yield base_get_process_output(baseargs + ['streams'])729        if 'Stream %s ' % location not in tmp:730            tmp = yield base_get_process_output(args)731            defer.returnValue(tmp)732        # Force p4base to be // in order to catch all changes to this client733        self._stream = location734        self.p4base = '//'735        # Check that our client references the stream736        tmp = yield base_get_process_output(baseargs + ['client', '-o', client])737        if 'Stream:\t%s' % location not in tmp:738            # Ensure the client exists739            p4clientcmd = '%s %s client' % (self.p4bin, ' '.join(baseargs))740            shargs = '%s -o %s | %s -i' % (p4clientcmd, client, p4clientcmd)741            tmp = yield utils.getProcessOutput('/bin/sh', ['-c', shargs])742            # Force switch the client stream743            tmp = yield base_get_process_output(baseargs + ['client', '-f', '-s', '-S', location, client])744        tmp = yield base_get_process_output(['-c', client] + args[:-1] + ['//%s%s' % (client, suffix)])745        defer.returnValue(tmp)746class P4Repository(Repository):747    ''' P4Repository handling '''748    def __init__(self, name, is_polling_enabled):749        super(P4Repository, self).__init__(name, is_polling_enabled)750    @staticmethod751    def config(port=None, user=None, password=None, client=None,752               binary=None, encoding=None, timezone=None, spec_options=None):753        ''' Common global p4 parameters '''754        # TODO : Add ticket management755        Scope.set_checked('p4_common_p4port', port, str)756        Scope.set_checked('p4_common_p4user', user, str)757        Scope.set_checked('p4_common_p4passwd', password, str)758        Scope.set_checked('p4_sync_p4client_spec_options', spec_options, str)759        Scope.set_checked('p4_sync_p4client', client, str)760        Scope.set_checked('p4_poll_p4bin', binary, str)761        Scope.set_checked('p4_poll_encoding', encoding, str)762        Scope.set_checked('p4_poll_server_tz', timezone, None)763    @staticmethod764    def add_views(*views):765        ''' Adds p4 mappings for current scope '''766        Scope.append('p4_sync_p4viewspec', *views)767    @staticmethod768    def set_stream(stream):769        ''' Adds p4 stream for current scope '''770        Scope.append('_p4stream', stream)771        # Hack the âViewâ entry. The view will be invalid, but it will be772        # overridden by the âStreamâ entry that we insert.773        # See master/buildbot/steps/source/p4.py in the buildbot sources774        # to understand why it works.775        Scope.append('p4_sync_p4viewspec',776                     ('//ignored/', '...\n\nStream: ' + stream + '\n\n#'))777    def get_sync_step(self, _, step_args):778        ''' Returns sync step for this repository '''779        return self._build_class(buildbot.steps.source.p4.P4,780                                 ('p4_common', 'p4_sync'),781                                 rendered=['p4_sync_p4client'],782                                 additional=step_args)783    def _build_change_sources(self, config, args):784        paths_to_poll = []785        for (depot_path, _) in self.get('p4_sync_p4viewspec', []):786            if depot_path.startswith('//'):787                # Get depot from //depot/788                base = depot_path[2:-1]789                paths_to_poll.append(base)790        for base in paths_to_poll:791            split_file = lambda branchfile: (None, branchfile)792            args['split_file'] = split_file793            args['p4base'] = '//' + base794            project_name = self.get_interpolated('project_name')795            if project_name is not None:796                args['project'] = project_name797            p4 = self._build_class(P4StreamSource,798                                   ('p4_common', 'p4_poll'),799                                   additional=args)800            yield p4801class GitRepository(Repository):802    ''' Git repository '''803    def __init__(self, name, repo_url, is_polling_enabled):804        super(GitRepository, self).__init__(name, is_polling_enabled)805        Scope.set_checked('git_common_repourl', repo_url, str)806    @staticmethod807    def config(git_bin=None,808               use_time_stamps=None,809               encoding=None,810               branch=None,811               submodules=True,812               shallow=None,813               progress=None,814               retry_fetch=None,815               clobber_on_failure=None,816               method=None):817        ''' Common global git parameters '''818        Scope.set_checked('git_poll_gitbin', git_bin, str)819        Scope.set_checked('get_poll_usetimestamps', use_time_stamps, bool)820        Scope.set_checked('git_poll_encoding', encoding, str)821        Scope.set_checked('git_common_branch', branch, str)822        Scope.set_checked('git_sync_submodules', submodules, bool)823        Scope.set_checked('git_sync_shallow', shallow, bool)824        Scope.set_checked('git_sync_progress', progress, bool)825        Scope.set_checked('git_sync_retryFetch', retry_fetch, bool)826        Scope.set_checked('git_sync_clobberOnFailure', clobber_on_failure, bool)827        Scope.set_checked('git_sync_method', method, str)828        assert method in [None, 'clobber', 'fresh', 'clean', 'copy']829    def get_sync_step(self, _, step_args):830        ''' Returns sync step for this repository '''831        return self._build_class(buildbot.steps.source.git.Git,832                                 ('git_common', 'git_sync'),833                                 additional=step_args)834    def _build_change_sources(self, config, args):835        project_name = self.get_interpolated('project_name')836        if project_name is not None:837            args['project'] = project_name838        yield self._build_class(buildbot.changes.gitpoller.GitPoller,839                                ('git_common', 'git_poll'),840                                additional=args)841class Step(Scope):842    ''' Build step '''843    def __init__(self, name):844        super(Step, self).__init__()845        self.properties['step_name'] = name846    @staticmethod847    def config(halt_on_failure=None,848               flunk_on_warnings=None,849               flunk_on_failure=None,850               warn_on_warnings=None,851               warn_on_failure=None,852               always_run=None,853               description=None,854               description_done=None,855               do_step_if=None,856               hide_step_if=None,857               workdir=None,858               timeout=None):859        ''' Fail behavior settings '''860        Scope.set_checked('step_haltOnFailure', halt_on_failure, bool)861        Scope.set_checked('step_flunkOnWarnings', flunk_on_warnings, bool)862        Scope.set_checked('step_flunkOnFailure', flunk_on_failure, bool)863        Scope.set_checked('step_warnOnWarnings', warn_on_warnings, bool)864        Scope.set_checked('step_warnOnFailure', warn_on_failure, bool)865        Scope.set_checked('step_alwaysRun', always_run, bool)866        Scope.set_checked('step_description', description, str)867        Scope.set_checked('step_descriptionDone', description_done, str)868        Scope.set_checked('step_doStepIf', do_step_if, None)869        Scope.set_checked('step_hideStepIf', hide_step_if, None)870        Scope.set_checked('step_workdir', workdir, str)871        Scope.set_checked('step_timeout', timeout, int)872    @abc.abstractmethod873    def _get_step(self, config, step_args):874        ''' Builds the buildbot step '''875        pass876    def _build(self, config):877        builder = self.get_parent_of_type(Builder)878        if builder is None:879            step_name = self.get_interpolated('step_name')880            msg = 'Step %s not declared in Builder scope' % step_name881            raise Exception(msg)882        step_args = self._get_prefixed_properties('step')883        builder.add_step(self._get_step(config, step_args))884class Sync(Step):885    ''' Syncs a previoulsy declared repository '''886    def __init__(self, repo_name):887        super(Sync, self).__init__('sync %s' % repo_name)888        self._repo_name = repo_name889    @staticmethod890    def config(mode=None,891               always_use_latest=None,892               retry=None,893               log_environ=None):894        ''' Common sync steps config '''895        Scope.set_checked('sync_mode', mode, str)896        assert mode in ['incremental', 'full', None]897        Scope.set_checked('sync_alwaysUseLatest', always_use_latest, bool)898        Scope.set_checked('sync_retry', retry, tuple)899        assert retry is None or len(retry) == 2900        Scope.set_checked('sync_logEnviron', log_environ, bool)901    def _get_step(self, config, step_args):902        repos = self.get_interpolated('source_control_repositories')903        if repos is None or self._repo_name not in repos:904            msg = 'Unable to find repository %s in scope.' % self._repo_name905            raise Exception(msg)906        step_args.update(self._get_prefixed_properties('sync'))907        return repos[self._repo_name].get_sync_step(config, step_args)908class Command(Step):909    ''' Executes a shell command '''910    def __init__(self, name, command):911        super(Command, self).__init__(name)912        self._command = shlex.split(command.strip())913    @staticmethod914    def config(want_stdout=None, want_stderr=None, lazy_log_files=None,915               max_time=None, interrupt_signal=None, sigterm_time=None,916               initial_stdin=None):917        Scope.set_checked('shell_command_want_stdout', want_stdout, bool)918        Scope.set_checked('shell_command_want_stderr', want_stderr, bool)919        Scope.set_checked('shell_command_lazylogfiles', lazy_log_files, bool)920        Scope.set_checked('shell_command_maxTime', max_time, int)921        assert interrupt_signal in [None, 'KILL', 'TERM']922        Scope.set_checked('shell_command_interruptSignal', interrupt_signal, str)923        Scope.set_checked('shell_command_sigterm_time', sigterm_time, int)924        Scope.set_checked('shell_command_initialStdin', initial_stdin, int)925    @staticmethod926    def set_decode_rc(return_value, meaning):927        ''' Adds a return code meaning to this command '''928        assert isinstance(return_value, int)929        buildbot_enum = {'success' : buildbot.status.results.SUCCESS,930                         'warnings' : buildbot.status.results.WARNINGS,931                         'error' : buildbot.status.results.FAILURE}932        assert meaning in buildbot_enum933        Scope.update('shell_command_decodeRC',934                     return_value,935                     buildbot_enum[meaning])936    @staticmethod937    def set_log_file(name, path):938        ''' Adds a log file to this command '''939        Scope.update('shell_command_logfiles', name, path)940    def _get_step(self, config, step_args):941        step_args['command'] = self.render(self._command)942        return self._build_class(buildbot.steps.shell.ShellCommand,943                                 'shell_command',944                                 rendered=['shell_command_logfiles'],945                                 additional=step_args)946class Pylint(Step):947    ''' Runs pylint '''948    def __init__(self, command):949        super(Pylint, self).__init__('pylint')950        self._command = command951    def _get_step(self, config, step_args):952        step_args['command'] = self.render(shlex.split(self._command))953        return self._build_class(buildbot.steps.python.PyLint,954                                 (),955                                 additional=step_args)956class Sphinx(Step):957    ''' Builds sphinx documentation '''958    def __init__(self):959        super(Sphinx, self).__init__('build sphink documentation')960    @staticmethod961    def config(build_dir=None,962               source_dir=None,963               builder=None,964               sphinx=None,965               tags=None,966               defines=None,967               mode=None):968        Scope.set_checked('sphinx_sphinx_builddir', build_dir, str)969        Scope.set_checked('sphinx_sphinx_sourcedir', source_dir, str)970        Scope.set_checked('sphinx_sphinx_builder', builder, str)971        Scope.set_checked('sphinx_sphinx', sphinx, str)972        Scope.set_checked('sphinx_tags', tags, str)973        Scope.set_checked('sphinx_defines', defines, str)974        Scope.set_checked('sphinx_mode', mode, str)975    def _get_step(self, config, step_args):976        return self._build_class(buildbot.steps.python.Sphinx,977                                 ('sphinx'),978                                 additional=step_args)979class _ChangeFilter(object):980    ''' Callable filtering change matching a regular expression against modified981        files982    '''983    def __init__(self, builder, project, accept=None, reject=None):984        self._builder = builder985        self._project = project986        self._accept = accept987        self._reject = reject988    def __call__(self, change):989        msg_prefix = 'ChangeFilter: checking change %s with %s' % (change.revision, self._builder)990        if self._project != change.project:991            log.msg('%s: not our project (%s != %s)' % (msg_prefix, self._project, change.project))992            return False993        if change.who.lower() == 'buildbot':994            log.msg('%s: ignoring user buildbot' % msg_prefix)995            return False996        if '[skip]' in change.comments.lower():997            log.msg('%s: ignoring [skip] tag' % msg_prefix)998            return False999        if self._accept == '.*' and self._reject is None:1000            log.msg('%s: accepted (accept rule is .*)' % msg_prefix)1001            return True1002        for file_path in change.files:1003            is_accepted = self._accept is None or re.match(self._accept, file_path) is not None1004            is_refused = self._reject is not None and re.match(self._reject, file_path) is not None1005            if is_accepted and not is_refused:1006                log.msg('%s: accepted (%s)' % (msg_prefix, file_path))1007                return True1008        log.msg('%s: no file matching %s and not matching %s' % (msg_prefix, self._accept, self._reject))1009        return False1010class Trigger(Step):1011    ''' Triggers builders declared in child scope '''1012    def __init__(self, name, *builder_names):1013        super(Trigger, self).__init__(name)1014        self._builder_names = []1015        self._builder_names.extend(builder_names)1016        self._nightly = None1017    @staticmethod1018    @contextlib.contextmanager1019    def builder(name, category=None, description=None):1020        ''' Helper to trigger a single builder '''1021        with Trigger('%s-trigger' % name):1022            with Builder(name, category, description) as builder:1023                yield builder1024    @staticmethod1025    def config(wait_for_finish=None,1026               always_use_latest=None):1027        Scope.set_checked('trigger_waitForFinish', wait_for_finish, bool)1028        Scope.set_checked('trigger_alwaysUseLatest', always_use_latest, bool)1029    def nightly(self,1030                minute=None,1031                hour=None,1032                day_of_month=None,1033                month=None,1034                day_of_week=None):1035        ''' Uses a nightly triggerable instead of a triggerable '''1036        self._nightly = {'minute' : minute,1037                         'hour' : hour,1038                         'dayOfMonth': day_of_month,1039                         'month' : month,1040                         'dayOfWeek' : day_of_week}1041    def add_builder(self, builder_name):1042        ''' Adds a builder to this trigger '''...configform.py
Source:configform.py  
...597      Sets the state of all the gui components on this ConfigForm to match the 598      contents of the given Configuration object.599      '''600      601      def set_checked( checkbox, checked ):602         self.__update_checklist.SetItemChecked( \603            self.__update_checklist.Items.IndexOf(checkbox), checked )604      605      # 1. --- set get the parts in the checklist box (data tab)606      set_checked(ConfigForm.__SERIES_CB, config.update_series_b)607      set_checked(ConfigForm.__NUMBER_CB, config.update_number_b)608      set_checked(ConfigForm.__PUBLISHED_CB, config.update_published_b)609      set_checked(ConfigForm.__RELEASED_CB, config.update_released_b)610      set_checked(ConfigForm.__TITLE_CB, config.update_title_b)611      set_checked(ConfigForm.__CROSSOVERS_CB, config.update_crossovers_b)612      set_checked(ConfigForm.__WRITER_CB, config.update_writer_b)613      set_checked(ConfigForm.__PENCILLER_CB, config.update_penciller_b)614      set_checked(ConfigForm.__INKER_CB, config.update_inker_b)615      set_checked(ConfigForm.__COVER_ARTIST_CB,config.update_cover_artist_b)616      set_checked(ConfigForm.__COLORIST_CB, config.update_colorist_b)617      set_checked(ConfigForm.__LETTERER_CB, config.update_letterer_b)618      set_checked(ConfigForm.__EDITOR_CB, config.update_editor_b)619      set_checked(ConfigForm.__SUMMARY_CB, config.update_summary_b)620      set_checked(ConfigForm.__IMPRINT_CB, config.update_imprint_b)621      set_checked(ConfigForm.__PUBLISHER_CB, config.update_publisher_b)622      set_checked(ConfigForm.__VOLUME_CB, config.update_volume_b)623      set_checked(ConfigForm.__CHARACTERS_CB, config.update_characters_b)624      set_checked(ConfigForm.__TEAMS_CB, config.update_teams_b)625      set_checked(ConfigForm.__LOCATIONS_CB, config.update_locations_b)626      set_checked(ConfigForm.__WEBPAGE_CB, config.update_webpage_b)627      628      # 2. --- then get the parts in the other checkboxes (options tab)629      self.__ow_existing_cb.Checked = config.ow_existing_b630      self.__convert_imprints_cb.Checked = config.convert_imprints_b631      self.__autochoose_series_cb.Checked = config.autochoose_series_b632      self.__confirm_issue_cb.Checked = config.confirm_issue_b633      self.__ignore_blanks_cb.Checked = config.ignore_blanks_b634      self.__download_thumbs_cb.Checked = config.download_thumbs_b635      self.__preserve_thumbs_cb.Checked = config.preserve_thumbs_b636      self.__fast_rescrape_cb.Checked = config.fast_rescrape_b637      self.__rescrape_notes_cb.Checked = config.rescrape_notes_b638      self.__rescrape_tags_cb.Checked = config.rescrape_tags_b639      self.__summary_dialog_cb.Checked = config.summary_dialog_b640      
...cleaning_utils.py
Source:cleaning_utils.py  
1import time2from core_data_modules.cleaners import Codes3from core_data_modules.data_models import Origin, Label4from core_data_modules.traced_data import Metadata5from core_data_modules.util import TimeUtils6class CleaningUtils(object):7    @staticmethod8    def make_label_from_cleaner_code(scheme, code, origin_id, origin_name="Pipeline Auto-Coder", date_time_utc=None,9                                     set_checked=False):10        """11        Constructs a new Label object from a code determined by a pipeline cleaner.12        :param scheme: Scheme which the `code` argument belongs to.13        :type scheme: core_data_modules.data_models.CodeScheme14        :param code: Code to construct the label from.15        :type code: Code16        :param origin_id: Identifier of the origin of this label.17        :type origin_id: str18        :param origin_name: Name of the origin of this label.19        :type origin_name: str20        :param date_time_utc: Date to set in the label as an ISO string in UTC, or None.21                              If None, uses the current system time in UTC.22        :type date_time_utc: str | None23        :param set_checked: Whether to set the `checked` property of the returned Label.24        :type set_checked: bool25        :return: A new label.26        :rtype: Label27        """28        if date_time_utc is None:29            date_time_utc = TimeUtils.utc_now_as_iso_string()30        origin = Origin(origin_id, origin_name, "External")31        return Label(scheme.scheme_id, code.code_id, date_time_utc, origin, checked=set_checked)32    @classmethod33    def apply_cleaner_to_text(cls, cleaner, text, scheme, set_checked=False):34        """35        Applies a cleaning function to a text, and returns a label if the cleaned value wasn't NC.36        :param cleaner: Cleaning function to apply.37        :type cleaner: function of str -> str38        :param text: Text to apply the cleaner to.39        :type text: str40        :param scheme: Scheme containing codes which the string returned from the `cleaner` can be matched against.41        :type scheme: core_data_modules.data_models.CodeScheme42        :param set_checked: Whether to set the `checked` property of the applied Label.43        :type set_checked: bool44        """45        clean_value = cleaner(text)46        # Don't label data which the cleaners couldn't code47        if clean_value == Codes.NOT_CODED:48            return None49        # Construct a label for the clean_value returned by the cleaner50        code_id = scheme.get_code_with_match_value(clean_value)51        origin_id = Metadata.get_function_location(cleaner)52        return cls.make_label_from_cleaner_code(scheme, code_id, origin_id, set_checked=set_checked)53    @classmethod54    def apply_cleaner_to_traced_data_iterable(cls, user, data, raw_key, clean_key, cleaner, scheme, set_checked=False):55        """56        Applies a cleaning function to an iterable of TracedData objects, updating each with a new Label object.57        :param user: Identifier of the user running this program, for TracedData Metadata.58        :type user: str59        :param data: TracedData objects to apply the cleaner to.60        :type data: iterable of TracedData61        :param raw_key: Key in each TracedData object of the raw text to be cleaned.62        :type raw_key: str63        :param clean_key: Key in each TracedData object to write cleaned labels to.64        :type clean_key: str65        :param cleaner: Cleaning function to apply to each TracedData[`raw_key`].66        :type cleaner: function of str -> str67        :param scheme: Scheme containing codes which the strings returned from the `cleaner` can be matched against.68        :type scheme: core_data_modules.data_models.CodeScheme69        :param set_checked: Whether to set the `checked` property of the applied Labels.70        :type set_checked: bool71        """72        for td in data:73            # Skip data that isn't present74            if raw_key not in td:75                continue76           77            label = cls.apply_cleaner_to_text(cleaner, td[raw_key], scheme, set_checked)78            if label is None:79                continue...system_manipulator.py
Source:system_manipulator.py  
1import sys, os, math2import configparser3import datetime4import graphene5sys.path.append(os.environ['SPACE_SHIP_HOME'] + '/api/background/mappers')6from system_mapper import SystemMapper7sys.path.append(os.environ['SPACE_SHIP_HOME'] + '/recital/native')8import mongo_native9config = configparser.ConfigParser()10config.read(os.environ['SPACE_SHIP_HOME'] + '/databases.config')11TIMESTAMP_PATTERN = os.environ.get('TIMESTAMP_PATTERN') or config['FORMATS']['timestamp']12class CreateSystem(graphene.Mutation):13    class Arguments:14        name = graphene.String()15        serialnumber = graphene.Float()16        launched = graphene.String()17        checked = graphene.String()18        state = graphene.String()19        supervisor = graphene.String()20        type = graphene.String()21    ok = graphene.Boolean()22    system = graphene.Field(lambda: SystemMapper)23    def mutate(self, info, name, serialnumber, launched, checked, state, supervisor, type):24        system = SystemMapper.init_scalar(mongo_native.create_system(\25            name, serialnumber, datetime.datetime.strptime(launched, TIMESTAMP_PATTERN), datetime.datetime.strptime(checked, TIMESTAMP_PATTERN), state, supervisor, type\26        ))27        ok = True28        return CreateSystem(system = system, ok = ok)29class RemoveSystem(graphene.Mutation):30    class Arguments:31        id = graphene.String()32    ok = graphene.Boolean()33    system = graphene.Field(lambda: SystemMapper)34    def mutate(self, info, id):35        system = SystemMapper.init_scalar(mongo_native.remove_system(id))36        ok = True37        return RemoveSystem(system = system, ok = ok)38class UpdateSystems(graphene.Mutation):39    class Arguments:40        id = graphene.String(default_value = '')41        name = graphene.String(default_value = '')42        serialnumber = graphene.Float(default_value = float('nan'))43        launched = graphene.String(default_value = '')44        checked = graphene.String(default_value = '')45        state = graphene.String(default_value = '')46        supervisor = graphene.String(default_value = '')47        type = graphene.String(default_value = '')48        set_name = graphene.String(default_value = '')49        set_serialnumber = graphene.Float(default_value = float('nan'))50        set_launched = graphene.String(default_value = '')51        set_checked = graphene.String(default_value = '')52        set_state = graphene.String(default_value = '')53        set_supervisor = graphene.String(default_value = '')54        set_type = graphene.String(default_value = '')55    ok = graphene.Boolean()56    systems = graphene.List(lambda: SystemMapper)57    def mutate(self, info, id, name, serialnumber, launched, checked, state, supervisor, type, 58        set_name, set_serialnumber, set_launched, set_checked, set_state, set_supervisor, set_type):59        systems = [SystemMapper.init_scalar(item) for item in mongo_native.update_systems(\60            _id = ObjectId(id) if id else None, name = name, 61            serialnumber = serialnumber if not math.isnan(serialnumber) else None, 62            launched = datetime.datetime.strptime(launched, TIMESTAMP_PATTERN) if launched else None, 63            checked = datetime.datetime.strptime(checked, TIMESTAMP_PATTERN) if checked else None, 64            state = ObjectId(state) if state else None, supervisor = ObjectId(supervisor) if supervisor else None, 65            type = ObjectId(type) if type else None,66            set_name = set_name, 67            set_serialnumber = set_serialnumber if set_serialnumber != float('nan') else None, 68            set_launched = datetime.datetime.strptime(set_launched, TIMESTAMP_PATTERN) if set_launched else None, 69            set_checked = datetime.datetime.strptime(set_checked, TIMESTAMP_PATTERN) if set_checked else None, 70            set_state = ObjectId(set_state) if set_state else None, set_supervisor = ObjectId(set_supervisor) if set_supervisor else None, 71            set_type = ObjectId(set_type) if set_type else None72        )]73        ok = True...LambdaTest’s Playwright tutorial will give you a broader idea about the Playwright automation framework, its unique features, and use cases with examples to exceed your understanding of Playwright testing. This tutorial will give A to Z guidance, from installing the Playwright framework to some best practices and advanced concepts.
Get 100 minutes of automation test minutes FREE!!
