How to use rebuild method in Behave

Best Python code snippet using behave

rebuilder.py

Source:rebuilder.py

...25        self.op_graph = kwargs['graph']26        self.rebuild_ops = kwargs['rebuild_ops']27    28    @abstractmethod29    def rebuild(self, configs=None):30        pass31    32    '''33    @abstractmethod34    def __get_rebuild_config(self, index, op_dict):35        pass36    37    def get_down_ops(self, op):38        down_ops = []39        for aop in self.op_graph.get_ops():40            if (aop.get_input_num() > 0):# and (op.get_name() in [bop.get_name for bop in get_input_ops_list(aop)]):41                aop_input_names = [bop.get_name() for bop in get_input_ops_list(aop)]42                if op.get_name() in aop_input_names:43                    down_ops.append(aop)44        return down_ops45    '''46class ReluRebuilder(BaseRebuilder):47    def __init__(self, *args, **kwargs):48        super().__init__(*args, **kwargs)49    50    def rebuild(self, graph, configs=None):51        if not isinstance(configs, dict):52            raise ValueError('Relu op rebuild configs should be dict')53        54        for relu_op in self.rebuild_ops:55            #input_ops = [node for node in self.op_graph.get_op(relu_op).get_input_ops()]56            input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(relu_op))]57            relu_rebuild_config = self.__get_rebuild_config(relu_op)58            59            #if relu_rebuild_config['relu_up_op'].get_type() != 'fix':60            input_ops[0].set_attr('fuse_relu', True)61            for down_op in relu_rebuild_config['relu_down_ops']:62                down_op.replace_input_ops(relu_rebuild_config['relu_op'], input_ops[0])63            graph.remove_op(relu_rebuild_config['relu_op'])64        65    def __get_rebuild_config(self, index):66        rebuild_config = {}67        rebuild_config['relu_op'] = self.op_graph.get_op(index)68        #rebuild_config['relu_up_op'] = self.op_graph.get_op(input_ops[0])69        70        relu_down_ops = get_down_ops(rebuild_config['relu_op'], self.op_graph)71        rebuild_config['relu_down_ops'] = relu_down_ops72                        73        return rebuild_config74class FixRebuilder(BaseRebuilder):75    def __init__(self, *args, **kwargs):76        super().__init__(*args, **kwargs)77    78    def rebuild(self, graph, configs=None):79        if not isinstance(configs, dict):80            raise ValueError('Fix op rebuild configs should be dict')81        82        #print(self.rebuild_ops)83        for fix_op in self.rebuild_ops:84            #print(fix_op)85            #input_ops = [node for node in self.op_graph.get_op(fix_op).get_input_ops()]86            input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(fix_op))]87            configs[input_ops[0].get_name()] = self.op_graph.get_op(fix_op).get_attrs()88            fix_rebuild_config = self.__get_rebuild_config(fix_op)89        90            for down_op in fix_rebuild_config['fix_down_ops']:91                down_op.replace_input_ops(fix_rebuild_config['fix_op'], input_ops[0])92            graph.remove_op(fix_rebuild_config['fix_op'])93        94    def __get_rebuild_config(self, index):95        rebuild_config = {}96        rebuild_config['fix_op'] = self.op_graph.get_op(index)97        #rebuild_config['fix_up_op'] = self.op_graph.get_op(op_dict[index][0])98        99        fix_down_ops = get_down_ops(rebuild_config['fix_op'], self.op_graph)100        rebuild_config['fix_down_ops'] = fix_down_ops101                        102        return rebuild_config103class MatmulRebuilder(BaseRebuilder):104    def __init__(self, *args, **kwargs):105        super().__init__(*args, **kwargs)106        107    def rebuild(self, graph, configs=None):108        if not isinstance(configs, dict):109            raise ValueError('Matmul op rebuild configs should be dict')110        for matmul_op in self.rebuild_ops:111            if not self.__should_rebuild(matmul_op):112                continue113            #input_ops = [node for node in self.op_graph.get_op(matmul_op).get_input_ops()]114            input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(matmul_op))]115            matmul_rebuild_config = self.__get_rebuild_config(matmul_op, input_ops)116            self.__rebuild_one_op(graph, matmul_rebuild_config, configs)117            118    def __should_rebuild(self, op):119        #if len(self.rebuild_ops[op]) == 3:120        if self.op_graph.get_op(op).get_input_num() == 3:121            return True122        else:123            return False124    125    def __get_rebuild_config(self, index, input_ops):126        rebuild_config = {}127        rebuild_config['matmul_op'] = self.op_graph.get_op(index)128        129        for node in input_ops:130            #node = self.op_graph.get_op(name)131            '''132            if node.get_type() == 'data':133                rebuild_config['input'] = node134            '''135            if (node.get_type() == 'const') and (node.get_output_tensor().ndim == 2):136                rebuild_config['weights'] = node137            elif (node.get_type() == 'const') and (node.get_output_tensor().ndim == 1):138                rebuild_config['bias'] = node139            else:140                rebuild_config['input'] = node141        142        matmul_down_ops = get_down_ops(rebuild_config['matmul_op'], self.op_graph)143        rebuild_config['matmul_down_ops'] = matmul_down_ops144        return rebuild_config145    146    def __rebuild_one_op(self, graph, rebuild_configs, fix_configs):147        matmul_op = rebuild_configs['matmul_op']148        149        wx_xt_ops: Dict[str, List[Op]] = {}150        wx_xt_ops['input'] = [rebuild_configs['input']]151        wx_xt_ops['input'].append(rebuild_configs['weights'])152        153        attrs: Dict[str, Any] = {}154        if matmul_op.get_type() == 'matmul':155            attrs['transpose_a'] = matmul_op.get_attr('transpose_a')156            attrs['transpose_b'] = matmul_op.get_attr('transpose_b')157        else:158            attrs['transpose_a'] = False159            attrs['transpose_b'] = True160        161        wx_xt_name = matmul_op.get_name() + '_matmul'162        wx_xt_mmul = graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)163        fix_configs[wx_xt_name] = fix_configs[matmul_op.get_name()]164        165        add_ops: Dict[str, List[Op]] = {}166        add_ops['input'] = [wx_xt_mmul]167        add_ops['input'].append(rebuild_configs['bias'])168        169        add_name = matmul_op.get_name() + '_eltwise'170        #add_name = matmul_op.get_name()171        #add_mmul = graph.create_op(add_name, 'eltwise', input_ops=add_ops)172        add_mmul = graph.create_op(add_name, 'add', input_ops=add_ops)173        fix_configs[add_name] = fix_configs[matmul_op.get_name()]174        175        if matmul_op.has_attr('fuse_relu'):176            add_mmul.set_attr('fuse_relu', True)177        178        for down_op in rebuild_configs['matmul_down_ops']:179            down_op.replace_input_ops(matmul_op, add_mmul)180        graph.remove_op(matmul_op)181class StridedSliceRebuilder(BaseRebuilder):182    def __init__(self, *args, **kwargs):183        super().__init__(*args, **kwargs)184        self.__fix_data = kwargs['fix_data']185        self.DATA_OPS = ['const']186        self.OPERATION_OPS = ['eltwise', 'matmul', 'linear', 'add']187    188    def rebuild(self, graph, configs=None):189        if not isinstance(configs, dict):190            raise ValueError('Strided_slice op rebuild configs should be dict')191        192        while len(self.rebuild_ops) > 0:193            #any_dict_index = list(self.rebuild_ops.keys())[0]194            any_index = self.rebuild_ops[0]195            stridedslice_rebuild_type, stridedslice_rebuild_config = self.__get_rebuild_config(any_index, self.rebuild_ops)196            self.__rebuild_one_op(graph, stridedslice_rebuild_type, stridedslice_rebuild_config, configs)197            198    def __get_rebuild_config(self, index, op_dict):199        rebuild_config = {}200        201        strided_op_list = []202        #input_op_name = op_dict[index][0]203        #input_ops = [node for node in self.op_graph.get_op(index).get_input_ops()]204        input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(index))]205        input_op_name = input_ops[0].get_name()206        207        for key in op_dict:208            #key_input_ops = [node for node in self.op_graph.get_op(key).get_input_ops()]209            key_input_ops = [node for node in get_input_ops_list(self.op_graph.get_op(key))]210            #if op_dict[key][0] == input_op_name:211            if key_input_ops[0].get_name() == input_op_name:212                strided_op_list.append(self.op_graph.get_op(key))213        rebuild_config['stridedslice_ops'] = strided_op_list214        215        stridedslice_up_op = self.op_graph.get_op(input_op_name)216        rebuild_config['stridedslice_up_op'] = stridedslice_up_op217        rebuild_type, rebuild_config['params_ops'] = self.__get_legal_stridedslice_param(stridedslice_up_op)218        219        stridedslice_down_ops = {}220        for op in strided_op_list:221            down_ops = get_down_ops(op, self.op_graph)222            stridedslice_down_ops[op] = down_ops223        rebuild_config['stridedslice_down_ops'] = stridedslice_down_ops224            225        for op in strided_op_list:226            #del op_dict[op.get_name()]227            op_dict.remove(op.get_name())228                        229        return rebuild_type, rebuild_config230    231    def __get_operation_stridedslice_param(self, up_op):232        param_ops_dict = {'input':None, 'weights':None, 'bias':None}233        234        input_num = up_op.get_input_num()235        #if (input_num == 2) and (up_op.get_type() == 'eltwise'):236        if (input_num == 2) and (up_op.get_type() == 'add'):237            input_ops = get_input_ops_list(up_op)238            input_ops_type = [input_op.get_type() for input_op in input_ops]239            if ('const' in input_ops_type) and (('matmul' in input_ops_type) or ('linear' in input_ops_type)):240            #if set(['const', 'matmul']) == set([input_op.get_type() for input_op in input_ops]):241                for op in input_ops:242                    if op.get_type() == 'const':243                        param_ops_dict['bias'] = op244                    elif op.get_type() in GLOBAL_VARIABLE.MATMUL_OPS:245                        if op.get_input_num() != 2:246                            raise ValueError('Can not handle this kind of strided_slice op')247                        for matmul_op in get_input_ops_list(op):248                            if op.get_type() == 'data':249                                param_ops_dict['input'] = matmul_op250                            else:251                                param_ops_dict['weights'] = matmul_op252                        253            else:254                raise ValueError('Can not handle this kind of strided_slice op')255            256        elif (input_num > 1) and (up_op.get_type() in GLOBAL_VARIABLE.MATMUL_OPS):257            param_ops = get_input_ops_list(up_op)258            for op in param_ops:259                if op.get_type() == 'data':260                    param_ops_dict['input'] = op261                elif op.get_output_tensor().ndim == 2:262                    param_ops_dict['weights'] = op263                elif op.get_output_tensor().ndim == 1:264                    param_ops_dict['bias'] = op265        else:266            raise ValueError('Can not handle this kind of strided_slice op')267        return param_ops_dict268        269    def __get_legal_stridedslice_param(self, up_op):270        if up_op.get_type() in self.DATA_OPS:271            stridedslice_type = 1272            param_ops_dict = {'const':up_op}273        elif up_op.get_type() in self.OPERATION_OPS:274            stridedslice_type = 2275            param_ops_dict = self.__get_operation_stridedslice_param(up_op)276        else:277            raise ValueError('Can not handle this kind of strided_slice op')278        return stridedslice_type, param_ops_dict279    280    def __rebuild_one_operation_op(self, graph, rebuild_configs, fix_configs):281        num = 0282        283        stridedslice_ops = rebuild_configs['stridedslice_ops']    284        stridedslice_up_op = rebuild_configs['stridedslice_up_op']285        params_ops = rebuild_configs['params_ops']286        param_input = params_ops['input']287        param_weights = params_ops['weights']288        param_bias = params_ops['bias']289        stridedslice_down_ops = rebuild_configs['stridedslice_down_ops']290        291        for op in stridedslice_ops:292            op_attrs = op.get_attrs()293            begin = op_attrs['begin']294            end = op_attrs['end']295            end_mask = op_attrs['end_mask']296            if param_weights.get_name() in self.__fix_data:297                weights_data = self.__fix_data[param_weights.get_name()]298            else:299                #weights_data = param_weights.get_attrs()['data']300                #weights_data = param_weights.get_attr('data')301                weights_data = const_op_data(param_weights)302            if end_mask == 1:303                weights_tensor = weights_data[begin[1]:end[1], :]304            else:305                weights_tensor = weights_data[begin[1]:end[1], begin[0]:end[0]]306            307            weight_op_name = param_weights.get_name()+'_split_'+str(num)308            #weights_op = graph.create_op(name=weight_op_name, kind=param_weights.get_type(), tensor=weights_tensor)309            weights_op = graph.create_const_op(weight_op_name, weights_tensor)310            fix_configs[weight_op_name] = fix_configs[param_weights.get_name()]           311 312            wx_xt_ops: Dict[str, List[Op]] = {}313            wx_xt_ops['input'] = [param_input]314            #wx_xt_ops['input'].append(param_input)315            wx_xt_ops['input'].append(weights_op)316            317            attrs: Dict[str, Any] = {}318            if stridedslice_up_op.get_type() == 'matmul':319                attrs['transpose_a'] = stridedslice_up_op.get_attr('transpose_a')320                attrs['transpose_b'] = stridedslice_up_op.get_attr('transpose_b')321            else:322                attrs['transpose_a'] = False323                attrs['transpose_b'] = True324            325            wx_xt_name = op.get_name() + '_matmul'326            wx_xt_mmul = graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)327            fix_configs[wx_xt_name] = fix_configs[op.get_name()]328            329            if param_bias is not None:330                if param_bias.get_name() in self.__fix_data:331                    bias_data = self.__fix_data[param_bias.get_name()]332                else:333                    #bias_data = param_bias.get_attrs()['data']334                    #bias_data = param_bias.get_attr('data')335                    bias_data = const_op_data(param_bias)336                bias_tensor =  bias_data[begin[1]:end[1]]337                bias_op_name = param_bias.get_name()+'_split_'+str(num)338                #bias_op = graph.create_op(name=bias_op_name, kind=param_bias.get_type(), tensor=bias_tensor)339                bias_op = graph.create_const_op(bias_op_name, bias_tensor)340                fix_configs[bias_op_name] = fix_configs[param_bias.get_name()]341                342                add_ops: Dict[str, List[Op]] = {}343                add_ops['input'] = [wx_xt_mmul]344                add_ops['input'].append(bias_op)345                add_name = op.get_name() + '_eltwise'346                #add_mmul = graph.create_op(add_name, 'eltwise', input_ops=add_ops)347                add_mmul = graph.create_op(add_name, 'add', input_ops=add_ops)348                fix_configs[add_name] = fix_configs[op.get_name()]349            350            for down_op in stridedslice_down_ops[op]:351                if param_bias is not None:352                    down_op.replace_input_ops(op, add_mmul)353                else:354                    down_op.replace_input_ops(op, wx_xt_mmul)355            graph.remove_op(op)356            num = num + 1357        graph.remove_op(stridedslice_up_op)358        359    def __rebuild_one_const_op(self, graph, rebuild_configs, fix_configs):360        num = 0361        362        stridedslice_ops = rebuild_configs['stridedslice_ops']    363        stridedslice_up_op = rebuild_configs['stridedslice_up_op']364        const_op = rebuild_configs['const']365        stridedslice_down_ops = rebuild_configs['stridedslice_down_ops']366        367        for op in stridedslice_ops:368            op_attrs = op.get_attrs()369            begin = op_attrs['begin']370            end = op_attrs['end']371            end_mask = op_attrs['end_mask']372            if const_op.get_name() in self.__fix_data:373                const_data = self.__fix_data[const_op.get_name()]374            else:375                #const_data = const_op.get_attrs()['data']376                #const_data = const_op.get_attr('data')377                const_data = const_op_data(const_op)378            if end_mask == 1:379                const_tensor = const_data[begin[1]:end[1], :]380            else:381                const_tensor = const_data[begin[1]:end[1], begin[0]:end[0]]382            383            split_op_name = const_op.get_name()+'_split_'+str(num)384            #split_op = graph.create_op(name=split_op_name, kind=const_op.get_type(), tensor=const_tensor)385            split_op = graph.create_const_op(split_op_name, const_tensor)386            fix_configs[split_op_name] = fix_configs[const_op.get_name()]           387 388            for down_op in stridedslice_down_ops[op]:389                down_op.replace_input_ops(op, split_op)390            graph.remove_op(op)391            num = num + 1392        graph.remove_op(stridedslice_up_op)393        394    395    def __rebuild_one_op(self, graph, rebuild_type, rebuild_configs, fix_configs):396        if rebuild_type == 1:397            self.__rebuild_one_const_op(graph, rebuild_configs, fix_configs)398        elif rebuild_type == 2:399            self.__rebuild_one_operation_op(graph, rebuild_configs, fix_configs)400        401        402'''403class StridedSliceRebuilder(Rebuilder):404    def __init__(self, *args, **kwargs):405        super(StridedSliceRebuilder).__init__(*args, **kwargs)406        407        408        self.graph = configs['graph']409        self.fix_data = configs['fix_data']410        self.stridedslice_ops = configs['stridedslice_ops']    411        self.stridedslice_up_op = configs['stridedslice_up_op']412        params_ops = configs['params_ops']413        self.param_input = params_ops['input']414        self.param_weights = params_ops['weights']415        self.param_bias = params_ops['bias']416        self.stridedslice_down_ops = configs['stridedslice_down_ops']417    418    def rebuild(self, fix_configs):419        num = 0420        for op in self.stridedslice_ops:421            op_attrs = op.get_attrs()422            begin = op_attrs['begin']423            end = op_attrs['end']424            end_mask = op_attrs['end_mask']425            if self.param_weights.get_name() in self.fix_data:426                weights_data = self.fix_data[self.param_weights.get_name()]427            else:428                #weights_data = self.param_weights.get_attrs()['data']429                weights_data = self.param_weights.get_attr('data')430            if end_mask == 1:431                weights_tensor = weights_data[begin[1]:end[1], :]432            else:433                weights_tensor = weights_data[begin[1]:end[1], begin[0]:end[0]]434            435            weight_op_name = self.param_weights.get_name()+'_split_'+str(num)436            weights_op = self.graph.create_op(name=weight_op_name, 437                                              kind=self.param_weights.get_type(), tensor=weights_tensor)438            fix_configs[weight_op_name] = fix_configs[self.param_weights.get_name()]           439 440            wx_xt_ops: Dict[str, List[Op]] = {}441            wx_xt_ops['input'] = [self.param_input]442            wx_xt_ops['input'].append(weights_op)443            444            attrs: Dict[str, Any] = {}445            attrs['transpose_a'] = False446            attrs['transpose_b'] = True447            448            wx_xt_name = op.get_name() + '_matmul'449            wx_xt_mmul = self.graph.create_op(wx_xt_name, 'matmul', attrs=attrs, input_ops=wx_xt_ops)450            fix_configs[wx_xt_name] = fix_configs[op.get_name()]451            452            if self.param_bias is not None:453                if self.param_bias.get_name() in self.fix_data:454                    bias_data = self.fix_data[self.param_bias.get_name()]455                else:456                    #bias_data = self.param_bias.get_attrs()['data']457                    bias_data = self.param_bias.get_attr('data')458                bias_tensor =  bias_data[begin[1]:end[1]]459                bias_op_name = self.param_bias.get_name()+'_split_'+str(num)460                bias_op = self.graph.create_op(name=bias_op_name, 461                                               kind=self.param_bias.get_type(), tensor=bias_tensor)462                fix_configs[bias_op_name] = fix_configs[self.param_bias.get_name()]463                464                add_ops: Dict[str, List[Op]] = {}465                add_ops['input'] = [wx_xt_mmul]466                add_ops['input'].append(bias_op)467                468                add_shape = wx_xt_mmul.get_output_tensor().dims469                add_tensor = np.zeros(add_shape, dtype=np.int64)470                471                add_name = op.get_name() + '_eltwise'472				#add_mmul = self.graph.create_op(add_name, 'eltwise', input_ops=add_ops, tensor=add_tensor)473                add_mmul = self.graph.create_op(add_name, 'add', input_ops=add_ops, tensor=add_tensor)474                fix_configs[add_name] = fix_configs[op.get_name()]475            476            for down_op in self.stridedslice_down_ops[op]:477                if self.param_bias is not None:478                    down_op.replace_input_ops(op, add_mmul)479                else:480                    down_op.replace_input_ops(op, wx_xt_mmul)481            self.graph.remove_op(op)482            num = num + 1483        self.graph.remove_op(self.stridedslice_up_op)484        485class FixRebuilder():486    def __init__(self, configs):487        self.graph = configs['graph']488        self.fix_op = configs['fix_op']    489        self.fix_up_op = configs['fix_up_op']490        self.fix_down_ops = configs['fix_down_ops']491    492    def rebuild(self):493        for down_op in self.fix_down_ops:494            down_op.replace_input_ops(self.fix_op, self.fix_up_op)495        self.graph.remove_op(self.fix_op)...

test_rebuild.py

Source:test_rebuild.py

1from mock import patch, MagicMock, mock2from srv.modules.runners import rebuild3class TestRebuild():4    """5    """6    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)7    @patch('salt.runner.Runner', autospec=True)8    @patch('salt.client.LocalClient', autospec=True)9    def test_minions(self, localclient, runner, mm):10        rebuild.__opts__ = {}11        local = localclient.return_value12        local.cmd.return_value = {'data1.ceph': 'data1.ceph'}13        rr = rebuild.Rebuild(['data1.ceph'])14        assert rr.minions == ['data1.ceph']15    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)16    @patch('salt.runner.Runner', autospec=True)17    @patch('salt.client.LocalClient', autospec=True)18    def test_minions_targets(self, localclient, runner, mm):19        rebuild.__opts__ = {}20        local = localclient.return_value21        local.cmd.return_value = {'data1.ceph': 'data1.ceph',22                                  'data2.ceph':'data2.ceph'}23        rr = rebuild.Rebuild(['I@roles:storage'])24        assert rr.minions == ['data1.ceph', 'data2.ceph']25    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)26    @patch('salt.runner.Runner', autospec=True)27    @patch('salt.client.LocalClient', autospec=True)28    def test_minions_multiple(self, localclient, runner, mm):29        rebuild.__opts__ = {}30        local = localclient.return_value31        local.cmd.return_value = {'data1.ceph': 'data1.ceph',32                                  'data2.ceph':'data2.ceph'}33        rr = rebuild.Rebuild(['data*.ceph', 'data*.ceph'])34        assert rr.minions == ['data1.ceph', 'data2.ceph']35    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)36    @patch('salt.runner.Runner', autospec=True)37    @patch('salt.client.LocalClient', autospec=True)38    def test_osd_list(self, localclient, runner, mm):39        rebuild.__opts__ = {}40        local = localclient.return_value41        local.cmd.return_value = {'data1.ceph': [0, 1, 2]}42        rr = rebuild.Rebuild(['data*.ceph'])43        result = rr._osd_list('data1.ceph')44        assert result == [0, 1, 2]45    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)46    @patch('salt.runner.Runner', autospec=True)47    @patch('salt.client.LocalClient', autospec=True)48    def test_osd_list_no_match(self, localclient, runner, mm):49        rebuild.__opts__ = {}50        local = localclient.return_value51        local.cmd.return_value = {'data1.ceph': [0, 1, 2]}52        rr = rebuild.Rebuild(['data*.ceph'])53        result = rr._osd_list('data2.ceph')54        assert result == None55    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)56    @patch('salt.runner.Runner', autospec=True)57    @patch('salt.client.LocalClient', autospec=True)58    def test_validate_osd_df(self, localclient, runner, mm):59        rebuild.__opts__ = {}60        local = localclient.return_value61        local.cmd.return_value = {}62        mm.return_value = 'admin'63        rr = rebuild.Rebuild(['data*.ceph'])64        osd_df = {'admin': {'nodes': '', 'summary':{'total_kb_avail': '0'}}}65        result = rr._validate_osd_df(osd_df)66        assert result == True67    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)68    @patch('salt.runner.Runner', autospec=True)69    @patch('salt.client.LocalClient', autospec=True)70    def test_validate_osd_df_missing_mm(self, localclient, runner, mm):71        rebuild.__opts__ = {}72        local = localclient.return_value73        local.cmd.return_value = {}74        mm.return_value = 'admin'75        rr = rebuild.Rebuild(['data*.ceph'])76        osd_df = {'': {'nodes': '', 'summary':{'total_kb_avail': '0'}}}77        result = rr._validate_osd_df(osd_df)78        assert result == False79    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)80    @patch('salt.runner.Runner', autospec=True)81    @patch('salt.client.LocalClient', autospec=True)82    def test_validate_osd_df_missing_nodes(self, localclient, runner, mm):83        rebuild.__opts__ = {}84        local = localclient.return_value85        local.cmd.return_value = {}86        mm.return_value = 'admin'87        rr = rebuild.Rebuild(['data*.ceph'])88        osd_df = {'admin': {'': '', 'summary':{'total_kb_avail': '0'}}}89        result = rr._validate_osd_df(osd_df)90        assert result == False91    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)92    @patch('salt.runner.Runner', autospec=True)93    @patch('salt.client.LocalClient', autospec=True)94    def test_validate_osd_df_missing_summary(self, localclient, runner, mm):95        rebuild.__opts__ = {}96        local = localclient.return_value97        local.cmd.return_value = {}98        mm.return_value = 'admin'99        rr = rebuild.Rebuild(['data*.ceph'])100        osd_df = {'admin': {'nodes': '', '':{'total_kb_avail': '0'}}}101        result = rr._validate_osd_df(osd_df)102        assert result == False103    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)104    @patch('salt.runner.Runner', autospec=True)105    @patch('salt.client.LocalClient', autospec=True)106    def test_validate_osd_df_missing_total(self, localclient, runner, mm):107        rebuild.__opts__ = {}108        local = localclient.return_value109        local.cmd.return_value = {}110        mm.return_value = 'admin'111        rr = rebuild.Rebuild(['data*.ceph'])112        osd_df = {'admin': {'nodes': '', 'summary':{'': '0'}}}113        result = rr._validate_osd_df(osd_df)114        assert result == False115    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)116    @patch('salt.runner.Runner', autospec=True)117    @patch('salt.client.LocalClient', autospec=True)118    def test_safe(self, localclient, runner, mm):119        rebuild.__opts__ = {}120        local = localclient.return_value121        local.cmd.return_value = {'admin': {'nodes': [122                                             {'id': 0, 'kb_used': 10},123                                             {'id': 1, 'kb_used': 10}124                                             ],125                                            'summary':{'total_kb_avail': 30}}}126        mm.return_value = 'admin'127        rr = rebuild.Rebuild(['data*.ceph'])128        result = rr.safe(['0', '1'])129        assert result == True130    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)131    @patch('salt.runner.Runner', autospec=True)132    @patch('salt.client.LocalClient', autospec=True)133    def test_safe_fails(self, localclient, runner, mm):134        rebuild.__opts__ = {}135        local = localclient.return_value136        local.cmd.return_value = {'admin': {'nodes': [137                                             {'id': 0, 'kb_used': 10},138                                             {'id': 1, 'kb_used': 10}139                                             ],140                                            'summary':{'total_kb_avail': 10}}}141        mm.return_value = 'admin'142        rr = rebuild.Rebuild(['data*.ceph'])143        result = rr.safe(['0', '1'])144        assert result == False145    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)146    @patch('salt.runner.Runner', autospec=True)147    @patch('salt.client.LocalClient', autospec=True)148    def test_safe_fails_validation(self, localclient, runner, mm):149        rebuild.__opts__ = {}150        local = localclient.return_value151        local.cmd.return_value = {'': {'nodes': [152                                             {'id': 0, 'kb_used': 10},153                                             {'id': 1, 'kb_used': 10}154                                             ],155                                            'summary':{'total_kb_avail': 10}}}156        mm.return_value = 'admin'157        rr = rebuild.Rebuild(['data*.ceph'])158        result = rr.safe(['0', '1'])159        assert result == False160    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)161    @patch('salt.runner.Runner', autospec=True)162    @patch('salt.client.LocalClient', autospec=True)163    def test_check_failed(self, localclient, runner, mm):164        rebuild.__opts__ = {}165        rr = rebuild.Rebuild(['data*.ceph'])166        ret = {0: True, 1:True}167        result = rr._check_failed(ret, 'data1.ceph')168        assert result == False169        assert rr.skipped == []170    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)171    @patch('salt.runner.Runner', autospec=True)172    @patch('salt.client.LocalClient', autospec=True)173    def test_check_failed_finds_failure(self, localclient, runner, mm):174        rebuild.__opts__ = {}175        rr = rebuild.Rebuild(['data*.ceph'])176        ret = {0: False, 1:True}177        result = rr._check_failed(ret, 'data1.ceph')178        assert result == True179        assert rr.skipped == ['data1.ceph']180    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)181    @patch('salt.runner.Runner', autospec=True)182    @patch('salt.client.LocalClient', autospec=True)183    def test_check_failed_finds_error(self, localclient, runner, mm):184        rebuild.__opts__ = {}185        rr = rebuild.Rebuild(['data*.ceph'])186        ret = "An error message"187        result = rr._check_failed(ret, 'data1.ceph')188        assert result == True189        assert rr.skipped == ['data1.ceph']190    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)191    @patch('salt.runner.Runner', autospec=True)192    @patch('salt.client.LocalClient', autospec=True)193    def test_run(self, localclient, runner, mm):194        rebuild.__opts__ = {}195        rr = rebuild.Rebuild(['data*.ceph'])196        rr._disengaged = mock.Mock()197        rr._disengaged.return_value = True198        rr.minions = ['data1.ceph']199        rr._osd_list = mock.Mock()200        rr._osd_list.return_value = [0, 1]201        rr.safe = mock.Mock()202        rr.safe.return_value = True203        rr._busy_wait = mock.Mock()204        rr.runner.cmd = mock.Mock()205        rr.runner.cmd.return_value = {}206        rr._check_failed = mock.Mock()207        rr._check_failed.return_value = False208        rr._skipped_summary = mock.Mock()209        result = rr.run()210        assert result == ""211        assert rr._osd_list.called212        assert rr.safe.called213        assert rr.runner.cmd.called214        assert rr._check_failed.called215        assert rr._skipped_summary.called216    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)217    @patch('salt.runner.Runner', autospec=True)218    @patch('salt.client.LocalClient', autospec=True)219    def test_run_multiple(self, localclient, runner, mm):220        rebuild.__opts__ = {}221        rr = rebuild.Rebuild(['data*.ceph'])222        rr._disengaged = mock.Mock()223        rr._disengaged.return_value = True224        rr.minions = ['data1.ceph', 'data2.ceph']225        rr._osd_list = mock.Mock()226        rr._osd_list.return_value = [0, 1]227        rr.safe = mock.Mock()228        rr.safe.return_value = True229        rr._busy_wait = mock.Mock()230        rr.runner.cmd = mock.Mock()231        rr.runner.cmd.return_value = {}232        rr._check_failed = mock.Mock()233        rr._check_failed.return_value = False234        rr._skipped_summary = mock.Mock()235        result = rr.run()236        assert result == ""237        assert rr._osd_list.call_count == 2238        assert rr.safe.call_count == 2239        assert rr.runner.cmd.call_count == 4240        assert rr._check_failed.call_count == 2241        assert rr._skipped_summary.called242    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)243    @patch('salt.runner.Runner', autospec=True)244    @patch('salt.client.LocalClient', autospec=True)245    def test_run_check_only(self, localclient, runner, mm):246        rebuild.__opts__ = {}247        rr = rebuild.Rebuild(['data*.ceph'])248        rr._disengaged = mock.Mock()249        rr._disengaged.return_value = True250        rr.minions = ['data1.ceph']251        rr._osd_list = mock.Mock()252        rr._osd_list.return_value = [0, 1]253        rr.safe = mock.Mock()254        rr.safe.return_value = True255        rr.runner.cmd = mock.Mock()256        rr.runner.cmd.return_value = {}257        rr._check_failed = mock.Mock()258        rr._check_failed.return_value = False259        rr._skipped_summary = mock.Mock()260        result = rr.run(checkonly=True)261        assert result == ""262        assert rr._osd_list.called263        assert rr.safe.called264        assert rr.runner.cmd.called is False265        assert rr._check_failed.called is False266        assert rr._skipped_summary.called267    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)268    @patch('salt.runner.Runner', autospec=True)269    @patch('salt.client.LocalClient', autospec=True)270    def test_run_safety_engaged(self, localclient, runner, mm):271        rebuild.__opts__ = {}272        rr = rebuild.Rebuild(['data*.ceph'])273        rr._disengaged = mock.Mock()274        rr._disengaged.return_value = False275        rr.minions = ['data1.ceph']276        rr._osd_list = mock.Mock()277        rr._osd_list.return_value = [0, 1]278        rr.safe = mock.Mock()279        rr.safe.return_value = True280        rr.runner.cmd = mock.Mock()281        rr.runner.cmd.return_value = {}282        rr._check_failed = mock.Mock()283        rr._check_failed.return_value = False284        rr._skipped_summary = mock.Mock()285        result = rr.run()286        assert result == ""287        assert rr._osd_list.called is False288        assert rr.safe.called is False289        assert rr.runner.cmd.called is False290        assert rr._check_failed.called is False291        assert rr._skipped_summary.called is False292    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)293    @patch('salt.runner.Runner', autospec=True)294    @patch('salt.client.LocalClient', autospec=True)295    def test_run_no_remove(self, localclient, runner, mm):296        rebuild.__opts__ = {}297        rr = rebuild.Rebuild(['data*.ceph'])298        rr._disengaged = mock.Mock()299        rr._disengaged.return_value = True300        rr.minions = ['data1.ceph']301        rr._osd_list = mock.Mock()302        rr._osd_list.return_value = []303        rr.safe = mock.Mock()304        rr.safe.return_value = True305        rr.runner.cmd = mock.Mock()306        rr.runner.cmd.return_value = {}307        rr._check_failed = mock.Mock()308        rr._check_failed.return_value = False309        rr._skipped_summary = mock.Mock()310        result = rr.run()311        assert result == ""312        assert rr._osd_list.called313        assert rr.safe.called is False314        assert rr.runner.cmd.called315        assert rr._check_failed.called is False316        assert rr._skipped_summary.called317    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)318    @patch('salt.runner.Runner', autospec=True)319    @patch('salt.client.LocalClient', autospec=True)320    def test_run_not_safe(self, localclient, runner, mm):321        rebuild.__opts__ = {}322        rr = rebuild.Rebuild(['data*.ceph'])323        rr._disengaged = mock.Mock()324        rr._disengaged.return_value = True325        rr.minions = ['data1.ceph']326        rr._osd_list = mock.Mock()327        rr._osd_list.return_value = [0, 1]328        rr.safe = mock.Mock()329        rr.safe.return_value = False330        rr.runner.cmd = mock.Mock()331        rr.runner.cmd.return_value = {}332        rr._check_failed = mock.Mock()333        rr._check_failed.return_value = False334        rr._skipped_summary = mock.Mock()335        result = rr.run()336        assert result == ""337        assert rr._osd_list.called338        assert rr.safe.called339        assert rr.runner.cmd.called is False340        assert rr._check_failed.called is False341        assert rr._skipped_summary.called is False342    @patch('srv.modules.runners.rebuild.master_minion', autospec=True)343    @patch('salt.runner.Runner', autospec=True)344    @patch('salt.client.LocalClient', autospec=True)345    def test_run_remove_failed(self, localclient, runner, mm):346        rebuild.__opts__ = {}347        rr = rebuild.Rebuild(['data*.ceph'])348        rr._disengaged = mock.Mock()349        rr._disengaged.return_value = True350        rr.minions = ['data1.ceph']351        rr._osd_list = mock.Mock()352        rr._osd_list.return_value = [0, 1]353        rr.safe = mock.Mock()354        rr.safe.return_value = True355        rr._busy_wait = mock.Mock()356        rr.runner.cmd = mock.Mock()357        rr.runner.cmd.return_value = {}358        rr._check_failed = mock.Mock()359        rr._check_failed.return_value = True360        rr._skipped_summary = mock.Mock()361        result = rr.run()362        assert result == ""363        assert rr._osd_list.called364        assert rr.safe.called365        assert rr.runner.cmd.called366        assert rr._check_failed.called...

cell.py

Source:cell.py

...124            rebuild_ops = self.__get_rebuild_ops(op_type)125            if (op_type == 'relu') and (len(rebuild_ops) > 0):                126                rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}127                rebuilder = ReluRebuilder(**rebuild_configs)128                rebuilder.rebuild(self.__graph, self.__fix_configs)129            130            elif (op_type == 'fix') and (len(rebuild_ops) > 0):131                for fix_op in rebuild_ops:132                    #input_ops = [op for op in self.__graph.get_op(fix_op).get_input_ops()]133                    input_ops = [op for op in get_input_ops_list(self.__graph.get_op(fix_op))]134                    #if self.__graph.get_op(op_input[fix_op][0]).get_type() == 'const':135                    if input_ops[0].get_type() == 'const':136                        self.__make_one_fix_data(input_ops[0].get_name(), fix_op)137                        #self.__make_one_fix_data(op_input[fix_op][0], fix_op)138                139                rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}140                rebuilder = FixRebuilder(**rebuild_configs)141                rebuilder.rebuild(self.__graph, self.__fix_configs)142                #self._fuse_fix_config()143                                144            elif (op_type == 'strided_slice') and (len(rebuild_ops) > 0):145                rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops, 'fix_data':self.__const_fix_data}146                rebuilder = StridedSliceRebuilder(**rebuild_configs)147                rebuilder.rebuild(self.__graph, self.__fix_configs)148                149            elif (op_type in GLOBAL_VARIABLE.MATMUL_OPS) and (len(rebuild_ops) > 0):150                rebuild_configs = {'graph':self.__graph, 'rebuild_ops':rebuild_ops}151                rebuilder = MatmulRebuilder(**rebuild_configs)152                rebuilder.rebuild(self.__graph, self.__fix_configs)153            154        self.__remove_invalid_const_data()155        '''156        for op in self.__graph.get_ops():157            print(op.get_name())158            print(op.get_type())159        '''160    '''161    def __rebuild_graph(self):162        for op_type in GLOBAL_VARIABLE.REBUILD_OPS_TYPE:163            if op_type == 'fix':164                fix_op_input = self.__get_op_input(op_type)165    166                for fix_op in fix_op_input:167                    self.__fix_configs[fix_op_input[fix_op]] = self.__graph.get_op(fix_op).get_attrs()168                    if self.__graph.get_op(fix_op_input[fix_op]).get_type() == 'const':169                        self.__make_one_fix_data(fix_op_input[fix_op], fix_op)170                    fix_rebuild_config = self.__get_fix_rebuild_config(fix_op, fix_op_input)171                    fix_rebuild_config['graph'] = self.__graph172                    fix_rebuilder = FixRebuilder(fix_rebuild_config)173                    fix_rebuilder.rebuild()174            if op_type == 'strided_slice':175                #stridedslice_op_input = self.__get_stridedslice_op_input__()176                stridedslice_op_input = self.__get_op_input(op_type)177                print(stridedslice_op_input)178                179                while len(stridedslice_op_input) > 0:180                    any_dict_index = list(stridedslice_op_input.keys())[0]181                    stridedslice_rebuild_config = self.__get_stridedslice_rebuild_config(any_dict_index, stridedslice_op_input)182                    stridedslice_rebuild_config['graph'] = self.__graph183                    stridedslice_rebuild_config['fix_data'] = self.__const_fix_data184                    stridedslice_rebuilder = StridedSliceRebuilder(stridedslice_rebuild_config)185                    stridedslice_rebuilder.rebuild(self.__fix_configs)186        self.__remove_invalid_const_data()187        for op in self.__graph.get_ops():188            print(op.get_name())189            print(op.get_type())190    '''    191    def reset(self, configs, xir_graph=None):192        pass193    194    def compile(self):195        parser_configs = {'ordered_nodes':self.__ordered_nodes,196                          'graph':self.__graph,197                          'fix_configs':self.__fix_configs,198                          'fix_data':self.__const_fix_data}199        xir_parser = XirParser(**parser_configs)...

rebuild.py

Source:rebuild.py

1#!/usr/bin/python32'''3  (C) Copyright 2018-2021 Intel Corporation.4  SPDX-License-Identifier: BSD-2-Clause-Patent5'''6from daos_core_base import DaosCoreBase7class DaosCoreTestRebuild(DaosCoreBase):8    # pylint: disable=too-many-ancestors9    """Run just the daos_test rebuild tests.10    :avocado: recursive11    """12    def test_rebuild_0to10(self):13        """Jira ID: DAOS-277014        Test Description:15            Run daos_test -r -s3 -u subtests=0-1016        Use cases:17            Core tests for daos_test rebuild18        :avocado: tags=all,pr,daily_regression19        :avocado: tags=hw,ib2,medium20        :avocado: tags=unittest21        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_0to1022        """23        self.run_subtest()24    def test_rebuild_12to15(self):25        """Jira ID: DAOS-277026        Test Description:27            Run daos_test -r -s3 -u subtests=12-1528        Use cases:29            Core tests for daos_test rebuild30        :avocado: tags=all,pr,daily_regression31        :avocado: tags=hw,ib2,medium32        :avocado: tags=unittest33        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_12to1534        """35        self.run_subtest()36    def test_rebuild_16(self):37        """Jira ID: DAOS-277038        Test Description:39            Run daos_test -r -s3 -u subtests=1640        Use cases:41            Core tests for daos_test rebuild42        :avocado: tags=all,pr,daily_regression43        :avocado: tags=hw,ib2,medium44        :avocado: tags=unittest45        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1646        """47        self.run_subtest()48    def test_rebuild_17(self):49        """Jira ID: DAOS-277050        Test Description:51            Run daos_test -r -s3 -u subtests=1752        Use cases:53            Core tests for daos_test rebuild54        :avocado: tags=all,pr,daily_regression55        :avocado: tags=hw,ib2,medium56        :avocado: tags=unittest57        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1758        """59        self.run_subtest()60    def test_rebuild_18(self):61        """Jira ID: DAOS-277062        Test Description:63            Run daos_test -r -s3 -u subtests=1864        Use cases:65            Core tests for daos_test rebuild66        :avocado: tags=all,pr,daily_regression67        :avocado: tags=hw,ib2,medium68        :avocado: tags=unittest69        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1870        """71        self.run_subtest()72    def test_rebuild_19(self):73        """Jira ID: DAOS-277074        Test Description:75            Run daos_test -r -s3 -u subtests=1976        Use cases:77            Core tests for daos_test rebuild78        :avocado: tags=all,pr,daily_regression79        :avocado: tags=hw,ib2,medium80        :avocado: tags=unittest81        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_1982        """83        self.run_subtest()84    def test_rebuild_20(self):85        """Jira ID: DAOS-277086        Test Description:87            Run daos_test -r -s3 -u subtests=2088        Use cases:89            Core tests for daos_test rebuild90        :avocado: tags=all,pr,daily_regression91        :avocado: tags=hw,ib2,medium92        :avocado: tags=unittest93        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_2094        """95        self.run_subtest()96    def test_rebuild_21(self):97        """Jira ID: DAOS-277098        Test Description:99            Run daos_test -r -s3 -u subtests=21100        Use cases:101            Core tests for daos_test rebuild102        :avocado: tags=all,pr,daily_regression103        :avocado: tags=hw,ib2,medium104        :avocado: tags=unittest105        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_21106        """107        self.run_subtest()108    def test_rebuild_22(self):109        """Jira ID: DAOS-2770110        Test Description:111            Run daos_test -r -s3 -u subtests=22112        Use cases:113            Core tests for daos_test rebuild114        :avocado: tags=all,pr,daily_regression115        :avocado: tags=hw,ib2,medium116        :avocado: tags=unittest117        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_22118        """119        self.run_subtest()120    def test_rebuild_23(self):121        """Jira ID: DAOS-2770122        Test Description:123            Run daos_test -r -s3 -u subtests=23124        Use cases:125            Core tests for daos_test rebuild126        :avocado: tags=all,pr,daily_regression127        :avocado: tags=hw,ib2,medium128        :avocado: tags=unittest129        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_23130        """131        self.run_subtest()132    def test_rebuild_24(self):133        """Jira ID: DAOS-2770134        Test Description:135            Run daos_test -r -s3 -u subtests=24136        Use cases:137            Core tests for daos_test rebuild138        :avocado: tags=all,pr,daily_regression139        :avocado: tags=hw,ib2,medium140        :avocado: tags=unittest141        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_24142        """143        self.run_subtest()144    def test_rebuild_25(self):145        """Jira ID: DAOS-2770146        Test Description:147            Run daos_test -r -s3 -u subtests=25148        Use cases:149            Core tests for daos_test rebuild150        :avocado: tags=all,pr,daily_regression151        :avocado: tags=hw,ib2,medium152        :avocado: tags=unittest153        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_25154        """155        self.run_subtest()156    def test_rebuild_26(self):157        """Jira ID: DAOS-2770158        Test Description:159            Run daos_test -r -s3 -u subtests=26160        Use cases:161            Core tests for daos_test rebuild162        :avocado: tags=all,pr,daily_regression163        :avocado: tags=hw,ib2,medium164        :avocado: tags=unittest165        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_26166        """167        self.run_subtest()168    def test_rebuild_27(self):169        """Jira ID: DAOS-2770170        Test Description:171            Run daos_test -r -s3 -u subtests=27172        Use cases:173            Core tests for daos_test rebuild174        :avocado: tags=all,pr,daily_regression175        :avocado: tags=hw,ib2,medium176        :avocado: tags=unittest177        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_27178        """179        self.run_subtest()180    def test_rebuild_28(self):181        """Jira ID: DAOS-2770182        Test Description:183            Run daos_test -r -s3 -u subtests=28184        Use cases:185            Core tests for daos_test rebuild186        :avocado: tags=all,pr,daily_regression187        :avocado: tags=hw,ib2,medium188        :avocado: tags=unittest189        :avocado: tags=daos_test,daos_core_test_rebuild,test_rebuild_28190        """...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.