How to use _merge_data method in lisa

Best Python code snippet using lisa_python

meta.py

Source:meta.py

...57                ret.append(string)58                break59    return ret60class BUImetaGrant(object):61    def _merge_data(self, d1, d2):62        """Merge data as list or dict recursively avoiding duplicates"""63        if not d1 and not d2:64            if isinstance(d1, dict) or isinstance(d2, dict):65                return {}66            return []67        if not d2:68            return d169        if not d1:70            return d271        if isinstance(d1, list) and isinstance(d2, list):72            return list(set(d1 + d2))73        if isinstance(d1, list) and not isinstance(d2, dict):74            if d2 in d1:75                return d176            return d1 + make_list(d2)77        if isinstance(d2, list) and not isinstance(d1, dict):78            if d1 in d2:79                return d280            return d2 + make_list(d1)81        if not isinstance(d1, dict) and not isinstance(d2, dict):82            if d1 == d2:83                return make_list(d1)84            else:85                return [d1, d2]86        res = d187        for key2, val2 in d2.items():88            if key2 in res:89                res[key2] = self._merge_data(val2, res[key2])90            else:91                res[key2] = val292        return res93    def _parse_clients(self, data, mode=None, parent=None):94        agents = clients = []95        advanced = {}96        if isinstance(data, list):97            if mode:98                if parent and parent not in PARSE_EXCLUDE_KEYS:99                    advanced[mode] = {parent: data}100                else:101                    advanced[mode] = {'clients': data}102            return data, agents, advanced103        if not isinstance(data, dict):104            if mode:105                if parent and parent not in PARSE_EXCLUDE_KEYS:106                    advanced[mode] = {parent: make_list(data)}107                else:108                    advanced[mode] = {'clients': make_list(data)}109            return make_list(data), agents, advanced110        for key, val in data.items():111            if key in PARSE_EXCLUDE_KEYS:112                continue113            cl1, ag1, ad1 = self._parse_clients(val, parent=key)114            agents = self._merge_data(agents, ag1)115            clients = self._merge_data(clients, cl1)116            agents = self._merge_data(agents, key)117            advanced = self._merge_data(advanced, ad1)118            advanced = self._merge_data(advanced, {key: cl1})119            if mode:120                if parent and parent not in PARSE_EXCLUDE_KEYS:121                    advanced = self._merge_data(advanced, {mode: {parent: cl1}})122                else:123                    advanced = self._merge_data(advanced, {mode: {key: cl1}})124        for key in ['clients'] + PARSE_RESERVED_KEYS:125            md = None126            if key in data:127                if key in PARSE_RESERVED_KEYS:128                    md = key129                    par = parent130                else:131                    par = key132                cl2, ag2, ad2 = self._parse_clients(data[key], md, parent=par)133                agents = self._merge_data(agents, ag2)134                if not md or md not in ['order', 'exclude']:135                    clients = self._merge_data(clients, cl2)136                if parent and parent not in PARSE_EXCLUDE_KEYS:137                    ro = ad2.get('ro')138                    rw = ad2.get('rw')139                    if ro and 'clients' in ro:140                        ro[parent] = ro['clients']141                        del ro['clients']142                        ad2['ro'] = ro143                    if rw and 'clients' in rw:144                        rw[parent] = rw['clients']145                        del rw['clients']146                        ad2['rw'] = rw147                advanced = self._merge_data(advanced, ad2)148        if 'agents' in data:149            ag3, cl3, ad3 = self._parse_agents(data['agents'])150            agents = self._merge_data(agents, ag3)151            clients = self._merge_data(clients, cl3)152            advanced = self._merge_data(advanced, ad3)153        return make_list(clients), make_list(agents), advanced154    def _parse_agents(self, data, mode=None):155        agents = clients = []156        advanced = {}157        if isinstance(data, list):158            if mode:159                advanced[mode] = {'agents': data}160            return data, clients, advanced161        if not isinstance(data, dict):162            if mode:163                advanced[mode] = {'agents': make_list(data)}164            return make_list(data), clients, advanced165        for key, val in data.items():166            if key in PARSE_EXCLUDE_KEYS:167                continue168            cl1, ag1, ad1 = self._parse_clients(data)169            agents = self._merge_data(agents, ag1)170            clients = self._merge_data(clients, cl1)171            agents = self._merge_data(agents, key)172            advanced = self._merge_data(advanced, ad1)173            if mode:174                advanced = self._merge_data(advanced, {mode: ad1})175            # FIXME: why did I do that?176            # advanced = self._merge_data(advanced, {key: cl1})177            # if mode:178            #     advanced = self._merge_data(advanced, {mode: {key: cl1}})179        for key in ['agents'] + PARSE_RESERVED_KEYS:180            md = None181            if key in data:182                if key in PARSE_RESERVED_KEYS:183                    md = key184                ag2, cl2, ad2 = self._parse_agents(data[key], md)185                agents = self._merge_data(agents, ag2)186                clients = self._merge_data(clients, cl2)187                advanced = self._merge_data(advanced, ad2)188        if 'clients' in data:189            cl3, ag3, ad3 = self._parse_clients(data['clients'])190            agents = self._merge_data(agents, ag3)191            clients = self._merge_data(clients, cl3)192            advanced = self._merge_data(advanced, ad3)193        return make_list(agents), make_list(clients), advanced194class BUIgrantHandler(BUImetaGrant, BUIacl):195    """This class is here to handle grants in a generic way.196    It will automatically merge grants from various backends that register to it197    """198    _id = 1199    _gp_admin_name = '@BUIADMINRESERVED'200    _gp_moderator_name = '@moderator'201    _gp_hidden = set([str(_gp_admin_name[1:]), str(_gp_moderator_name[1:])])202    _grants = {}203    _groups = {}204    _options = {}205    _backends = {}206    _name = 'meta_grant'207    @property208    def id(self):209        """current handler id, used to detect configuration changes"""210        return self._id211    @property212    def grants(self):213        """grants managed by our handler"""214        return self._grants215    @property216    def groups(self):217        """groups managed by our handler"""218        return self._groups219    @property220    def options(self):221        """options of our ACL engine"""222        return self._options223    @options.setter224    def options(self, value):225        """set the options of our engine"""226        self._options = value227        if self._options.get('legacy'):228            self._options['extended'] = False229    def changed(self, sid):230        """detect a configuration change"""231        return sid != self._id232    def reset(self, reset_from):233        """a configuration change occurred, we reload our grants and groups"""234        self._grants.clear()235        self._groups.clear()236        self._reset_cached()237        self._id += 1238        for name, backend in self._backends.items():239            if name == reset_from:240                continue241            backend.reload()242    def opt(self, key, default=False):243        """access a given option"""244        if key not in self.options:245            return default246        return self.options.get(key)247    def register_backend(self, name, backend):248        """register a new ACL backend249        :param name: Backend name250        :type name: str251        :param backend: ACL Backend252        :type backend: :class:`burpui.misc.acl.interface.BUIaclLoader`253        """254        self._backends[name] = backend255    def set_grant(self, name, grant):256        """parse and set the given grants"""257        if name in self._grants:258            return self._grants[name].add_grants(grant)259        self._grants[name] = BUIaclGrant(name, grant)260        return self._grants[name].grants261    def set_group(self, name, members):262        """parse and set the given group"""263        if name in self._groups:264            return self._groups[name].add_members(members)265        self._groups[name] = BUIaclGroup(name, members)266        return self._groups[name].members267    def set_admin(self, admins):268        """parse and set the admins"""269        self.set_group(self._gp_admin_name, admins)270    def set_moderator(self, moderators):271        """parse and set the moderators"""272        self.set_group(self._gp_moderator_name, moderators)273    def set_moderator_grants(self, grants):274        """parse and set the moderators grants"""275        self.set_grant(self._gp_moderator_name, grants)276    def get_member_groups(self, member):277        groups = []278        for group in self._groups.values():279            (ret, inh) = group.is_member(member)280            if ret and group.name not in self._gp_hidden:281                groups.append((group.name, inh))282        return groups283    def _gen_key(self, username):284        return '{}-{}'.format(self._name, username)285    def _set_cached(self, username, value):286        key = self._gen_key(username)287        return cache.cache.set(key, value)288    def _get_cached(self, username):289        key = self._gen_key(username)290        return cache.cache.get(key)291    def _reset_cached(self):292        cache.clear()293    def _is_cached(self, username):294        key = self._gen_key(username)295        return cache.cache.has(key)296    def _extract_grants(self, username, parent=None):297        if not self._is_cached(username):298            data = {}299            if username in self.grants:300                grants = self.grants[username].grants301            else:302                grants = []303            clients, agents, advanced = self._parse_clients(grants)304            data['clients'] = clients305            data['agents'] = agents306            data['advanced'] = [advanced] if advanced else []307            def __merge_grants_with(grp, prt):308                data2 = self._extract_grants(grp, prt)309                data['clients'] = self._merge_data(310                    data['clients'],311                    data2['clients']312                )313                data['agents'] = self._merge_data(314                    data['agents'],315                    data2['agents']316                )317                tmp = data2['advanced']318                if tmp:319                    data['advanced'] += tmp320            # moderator is also a group321            for gname, group in self.groups.items():322                # no grants need to be parsed for admins323                if gname == self._gp_admin_name:324                    continue325                (ret, _) = group.is_member(username)326                if not parent:327                    parent = set([username])328                elif isinstance(parent, set):329                    parent.add(username)330                if ret and gname != username and parent and gname not in parent:331                    __merge_grants_with(gname, parent)332            self._set_cached(username, data)333            return data334        return self._get_cached(username)335    def _extract_clients(self, username):336        ret = self._extract_grants(username)337        return ret.get('clients', [])338    def _extract_agents(self, username):339        ret = self._extract_grants(username)340        return ret.get('agents', [])341    def _extract_advanced(self, username, idx=None):342        ret = self._extract_grants(username).get('advanced', [])343        if idx is not None:344            return ret[idx]345        if self.opt('inverse_inheritance'):346            return reversed(ret)347        return ret348    def _extract_advanced_mode(self, username, mode, kind, idx):349        return self._extract_advanced(username, idx).get(mode, {}).get(kind, [])350    def _client_match(self, username, client):351        clients = self._extract_clients(username)352        if not clients:353            return None354        if self.opt('extended'):355            matches = []356            for exp in clients:357                regex = fnmatch.translate(exp)358                if re.match(regex, client):359                    matches.append(exp)360            return matches if matches else False361        else:362            return [client] if client in clients else False363    def _server_match(self, username, server):364        servers = self._extract_agents(username)365        if not servers:366            return None367        if self.opt('extended'):368            matches = []369            for exp in servers:370                regex = fnmatch.translate(exp)371                if re.match(regex, server):372                    matches.append(exp)373            return matches if matches else False374        else:375            return [server] if server in servers else False376    # implement BUIacl methods377    def is_admin(self, username):378        """See :func:`burpui.misc.acl.interface.BUIacl.is_admin`"""379        if self._gp_admin_name in self._groups:380            return self._groups[self._gp_admin_name].is_member(username)381        return False, None382    def is_moderator(self, username):383        """See :func:`burpui.misc.acl.interface.BUIacl.is_moderator`"""384        if self._gp_moderator_name in self._groups:385            return self._groups[self._gp_moderator_name].is_member(username)386        return False, None387    def is_client_rw(self, username=None, client=None, server=None):388        """See :func:`burpui.misc.acl.interface.BUIacl.is_client_rw`"""389        if not username or not client:  # pragma: no cover390            return False391        (is_admin, _) = self.is_admin(username)392        ret = is_admin or self.opt('assume_rw', True) or self.opt('legacy')393        if self.is_client_allowed(username, client, server):394            # legacy mode: assume rw for everyone395            if self.opt('legacy'):396                return True397            client_match = self._client_match(username, client)398            advanced = self._extract_advanced(username)399            if client_match is None and username == client:400                client_match = [username]401            if server:402                server_match = self._server_match(username, server)403                if not server_match and not client_match:404                    return is_admin or self.opt('assume_rw', True)405                for adv in advanced:406                    order = _extract_key(adv, 'order', [server] + server_match, DEFAULT_EVAL_ORDER)407                    for adv2 in advanced:408                        # the whole agent is rw and we did not find explicit entry for409                        # client_match410                        if client_match is False:411                            if server_match and \412                                    any(x in adv.get('rw', {}) or413                                        x in adv.get('rw', {}).get('agents', [])414                                        for x in server_match):415                                return True416                            if server in adv.get('rw', {}) or \417                                    server in adv.get('rw', {}).get('agents', []):418                                return True419                        if server_match and \420                                any(x in adv.get('rw', {}) or421                                    x in adv.get('rw', {}).get('agents', [])422                                    for x in server_match):423                            for odr in order:424                                if client_match and \425                                    any(x in adv2.get(odr, []) or426                                        x in adv2.get(odr, {}).get('clients', []) or427                                        any(x in adv2.get(odr, {}).get(y, [])428                                            for y in server_match)429                                        for x in client_match430                                        ):431                                    return MODE_RETURN.get(odr, False)432                        if server_match and \433                                any(x in adv.get('ro', {}) or434                                    x in adv.get('ro', {}).get('agents', [])435                                    for x in server_match):436                            for odr in order:437                                if client_match and \438                                    any(x in adv2.get(odr, {}).get('clients', []) or439                                        x in adv2.get(odr, []) or440                                        any(x in adv2.get(odr, {}).get(y, [])441                                            for y in server_match)442                                        for x in client_match443                                        ):444                                    return MODE_RETURN.get(odr, False)445            for adv in advanced:446                if server:447                    key = [server] + self._server_match(username, server)448                else:449                    key = None450                order = _extract_key(adv, 'order', key, DEFAULT_EVAL_ORDER)451                for odr in order:452                    eval_clients = adv.get(odr, {}).get('clients', [])453                    if client_match and \454                            any(x in eval_clients for x in client_match):455                        return MODE_RETURN.get(odr, False)456                    if client and \457                            client in eval_clients:458                        return MODE_RETURN.get(odr, False)459        return ret460    def is_client_allowed(self, username=None, client=None, server=None):461        """See :func:`burpui.misc.acl.interface.BUIacl.is_client_allowed`"""462        if not username or not client:  # pragma: no cover463            return False464        (is_admin, _) = self.is_admin(username)465        client_match = self._client_match(username, client)466        if client_match is None and username == client:467            client_match = [username]468        elif not client_match:469            client_match = False470        if server:471            server_match = self._server_match(username, server)472            if server_match is not None or self.opt('legacy'):473                if not server_match:474                    return is_admin475                advanced = self._extract_advanced(username)476                if self.opt('implicit_link', True) and not advanced:477                    advanced = False478                if advanced is not False:479                    for idx, adv in enumerate(advanced):480                        order = _extract_key(adv, 'order', [server] + server_match, DEFAULT_EVAL_ORDER)481                        excludes = _extract_key(adv, 'exclude', [server] + server_match, fallback='agents')482                        if all(x not in adv for x in server_match):483                            for odr in order:484                                if odr == 'exclude' and (485                                        any(x in excludes for x in client_match) or486                                        _glob_match(excludes, client, self.opt('extended'))):487                                    return False488                                elif any(x in y489                                         for x in server_match490                                         for y in self._extract_advanced_mode(username, odr, 'agents', idx)491                                         ):492                                    return True493                        tmp = set(adv.get(server, []))494                        for srv in server_match:495                            tmp |= set(adv.get(srv, []))496                        adv2 = list(tmp)497                        excludes = _extract_key(adv, 'exclude', [server] + server_match)498                        for odr in order:499                            if odr == 'exclude' and (500                                    any(x in excludes for x in client_match) or501                                    _glob_match(excludes, client, self.opt('extended'))):502                                return False503                            elif client_match is not False and \504                                    (any(x in adv2 for x in client_match) or505                                     client in adv2):506                                return True507                    return False508        advanced = self._extract_advanced(username)509        if advanced:510            for adv in advanced:511                order = _extract_key(adv, 'order', None, DEFAULT_EVAL_ORDER)512                excludes = _extract_key(adv, 'exclude', None)513                for odr in order:514                    if odr == 'exclude' and client_match and (515                            any(x in excludes for x in client_match) or516                            _glob_match(excludes, client, self.opt('extended'))):517                        return False518        return client_match is not False or is_admin519    def is_server_rw(self, username=None, server=None):520        """See :func:`burpui.misc.acl.interface.BUIacl.is_server_rw`"""521        if not username:  # pragma: no cover522            return False523        # special case single-agent mode524        if not server and config.get('STANDALONE'):525            server = 'local'526        (is_admin, _) = self.is_admin(username)527        ret = is_admin or self.opt('assume_rw', True) or self.opt('legacy')528        if self.is_server_allowed(username, server):529            server_match = self._server_match(username, server)530            if not server_match:531                return is_admin or self.opt('assume_rw', True)532            advanced = self._extract_advanced(username)533            for adv in advanced:534                order = _extract_key(adv, 'order', [server] + server_match, DEFAULT_EVAL_ORDER)535                for odr in order:536                    if any(x in adv.get(odr, {}).get('agents', []) for x in server_match):537                        return MODE_RETURN.get(odr, False)538        return ret539    def is_server_allowed(self, username=None, server=None):540        """See :func:`burpui.misc.acl.interface.BUIacl.is_server_allowed`"""541        if not username or not server:542            return False543        server_match = self._server_match(username, server)544        (is_admin, _) = self.is_admin(username)545        if server_match is None and self.opt('legacy'):546            server_match = False547        return server_match is not False or is_admin548class BUIaclGroup(object):549    """The :class:`burpui.misc.acl.interface.BUIaclGroup` class is used to550    represent a Group"""551    def __init__(self, name, members=None):552        self._name = name553        self._set_members(members)554        self.has_subgroups = -1555    def _parse_members(self, members):556        # we support only lists557        if members and ',' in members and not isinstance(members, list):558            parsed = [x.strip() for x in members.split(',')]559        else:560            parsed = make_list(members)561        return parsed562    def _set_members(self, members):563        self._members = set(self._parse_members(members))564    def add_members(self, new_members):565        new_members = self._parse_members(new_members)566        self._members = self._members | set(new_members)567        # reset the flag568        self.has_subgroups = -1569        return new_members570    def del_members(self, members_remove):571        members_remove = self._parse_members(members_remove)572        self._members = self._members - set(members_remove)573        # reset the flag574        self.has_subgroups = -1575    def is_member(self, member, parent=None):576        inherit = set()577        ret = member in self._members578        if not ret and (self.has_subgroups > 0 or self.has_subgroups == -1):579            self.has_subgroups = 0580            for mem in self.members:581                # avoid infinite loop with mutual inheritance582                if parent and mem in parent:583                    continue584                if mem.startswith('@'):585                    self.has_subgroups += 1586                    if mem in meta_grants._groups:587                        if parent:588                            parent.append(mem)589                        else:590                            parent = [mem]591                        (ret, inh2) = meta_grants._groups[mem].is_member(member, parent=parent)592                        if ret:593                            for subinh in inh2:594                                inherit.add(subinh)595                            inherit.add(mem)596                            # no break, we may have other inheritance at the level597        return ret, list(inherit)598    @property599    def name(self):600        if self._name and any(self._name.startswith(x) for x in ['@', '+']):601            return str(self._name[1:])602        return self._name603    @property604    def members(self):605        return list(self._members)606class BUIaclGrant(BUImetaGrant):607    """The :class:`burpui.misc.acl.interface.BUIaclGrant` class is used to608    represent a Grant"""609    def __init__(self, name, grants):610        self._name = name611        self._grants = self._parse_grants(grants)612    def _parse_grants(self, grants):613        try:614            ret = json.loads(grants)615        except (ValueError, TypeError):616            # handle empty/missing grants617            if not grants:618                return []619            # ignore mal-formatted json620            if any(x in grants for x in ['{', '}', '[', ']']):621                ret = None622            elif grants and ',' in grants:623                ret = [x.rstrip() for x in grants.split(',')]624            else:625                ret = make_list(grants)626        return ret627    @property628    def name(self):629        if self._name and any(self._name.startswith(x) for x in ['@', '+']):630            return str(self._name[1:])631        return self._name632    @property633    def grants(self):634        return self._grants635    @property636    def grants_raw(self):637        return json.dumps(self._grants)638    def add_grants(self, grants):639        parsed = self._parse_grants(grants)640        self._grants = self._merge_data(self._grants, parsed)641        return parsed...

chuan_bi_data.py

Source:chuan_bi_data.py

...11            ).reset_index(drop=True)12        return new_dataframe.rename(columns={'int_id_col_name': id_col_name})13    def _dropna(self, data, column, axis):14        return data.dropna(column, axis=axis)15    def _merge_data(self, left_data, left_key, right_data, right_key, how):16        return left_data.merge(17            right_data,18            how=how,19            left_on=left_key,20            right_on=right_key)21    def _group_tags(self, data, group_by, tag_column):22        return data.groupby(23            [group_by])[tag_column].apply(24            ','.join).reset_index()25    def _merge_cv_datasets(26        self,27        professionals,students,28        questions,answers,29        tags,tag_questions,tag_users, questions_score):30        31      32        tag_question = self._merge_data(33            left_data=tag_questions,34            left_key='tag_questions_tag_id',35            right_data=tags,36            right_key='tags_tag_id',37            how='inner')38        tag_question = self._group_tags(39            data=tag_question,40            group_by='tag_questions_question_id',41            tag_column='tags_tag_name')42        43        tag_question = tag_question.rename(44            columns={'tags_tag_name': 'questions_tag_name'})45        46        tags_pro = self._merge_data(47            left_data=tag_users,48            left_key='tag_users_tag_id',49            right_data=tags,50            right_key='tags_tag_id',51            how='inner')52        tags_pro = self._group_tags(53            data=tags_pro,54            group_by='tag_users_user_id',55            tag_column='tags_tag_name')56        tags_pro = tags_pro.rename(57            columns={'tags_tag_name': 'professionals_tag_name'})58        59        questions = self._merge_data(60            left_data=questions,61            left_key='questions_id',62            right_data=tag_question,63            right_key='tag_questions_question_id',64            how='left')65        professionals = self._merge_data(66            left_data=professionals,67            left_key='professionals_id',68            right_data=tags_pro,69            right_key='tag_users_user_id',70            how='left')71        72        questions = self._merge_data(73            left_data=questions,74            left_key='questions_id',75            right_data=questions_score,76            right_key='id',77            how='left')78        79        questions = self._merge_data(80            left_data=questions,81            left_key='questions_author_id',82            right_data=students,83            right_key='students_id',84            how='left')85        86        merge = self._merge_data(87            left_data=answers,88            left_key='answers_question_id',89            right_data=questions,90            right_key='questions_id',91            how='inner')92        93        merge = self._merge_data(94            left_data=merge,95            left_key='answers_author_id',96            right_data=professionals,97            right_key='professionals_id',98            how='inner')99        100        return questions, professionals, merge101  102    def _drop_duplicates_tags(self, data, col_name):103        return (104            data[col_name].str.split(105                ',').apply(set).str.join(','))106    def _merge_pro_pre_ans_tags(self, professionals, merge):107        professionals_prev_ans_tags = (108            merge[['professionals_id', 'questions_tag_name']])109        professionals_prev_ans_tags = professionals_prev_ans_tags.dropna()110        111        professionals_prev_ans_tags = self._group_tags(112            data=professionals_prev_ans_tags,113            group_by='professionals_id',114            tag_column='questions_tag_name')115        116        professionals_prev_ans_tags['questions_tag_name'] = \117        self._drop_duplicates_tags(118            professionals_prev_ans_tags, 'questions_tag_name')119        120        professionals = self._merge_data(121            left_data=professionals,122            left_key='professionals_id',123            right_data=professionals_prev_ans_tags,124            right_key='professionals_id',125            how='left')126        127        professionals['professional_all_tags'] = (128            professionals[['professionals_tag_name',129                           'questions_tag_name']].apply(130                lambda x: ','.join(x.dropna()),131                axis=1))132        return professionals133    def prepare(134        self,...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.