Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use getStructure method in fMBT

Best Python code snippet using fMBT_python

backpropagation.py

Source:backpropagation.py

...44            lastProgressUpdate = 045            totalTrainingCost = 046            for miniBatch in range(int(self.__inputSize / self.__miniBatchSize)):47                deltaSum = [np.array([0])]48                for layerSize in self.__network.getStructure()[1:]:49                    deltaSum.append(np.zeros(layerSize))50                grads = []51                for layer in range(len(self.__network.getStructure())-1):52                    grads.append(np.zeros((self.__network.getStructure()[layer+1], self.__network.getStructure()[layer])))5354                for trainingExample in range(self.__miniBatchSize):55                    currentIndex = miniBatch * self.__miniBatchSize + trainingExample56                    percentProgress = currentIndex / (int(self.__inputSize / self.__miniBatchSize) * self.__miniBatchSize)57                    if percentProgress > lastProgressUpdate + 0.005:58                        self.__UIelements.updateProgressBar(percentProgress*100)59                        lastProgressUpdate = percentProgress60                    61                    # forward calculation62                    trainingImage, actualLabel = self.__getTrainingExample(trainingIndices[currentIndex])63                    exampleCost = self.__exampleCost(trainingImage.reshape(784), actualLabel)64                    totalTrainingCost += exampleCost6566                    # back propagation of errors67                    deltas = [np.array([0])]68                    for layerSize in self.__network.getStructure()[1:]:69                        deltas.append(np.zeros(layerSize))70                    targetOutputActivations = np.zeros(self.__network.getStructure()[-1])71                    targetOutputActivations[actualLabel] = 172                    outputActivations = self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))73                    deltas[len(self.__network.getStructure())-1] = (outputActivations - targetOutputActivations) * (outputActivations) * (1 - outputActivations)74                    for layer in range(len(self.__network.getStructure())-1,0,-1):75                        layerActivation = self.__network.getNeuronActivation(layer-1, range(self.__network.getStructure()[layer-1]))76                        deltas[layer-1] = (np.transpose(self.__network.getConnectionWeights(layer-1)) @ deltas[layer]) * (layerActivation) * (1 - layerActivation)77                    78                    # accumulators79                    for layer in range(len(self.__network.getStructure())-1):80                        deltaSum[layer+1] += deltas[layer+1]81                        grads[layer] += deltas[layer+1].reshape(len(deltas[layer+1]),1) * self.__network.getNeuronActivation(layer, range(self.__network.getStructure()[layer]))82                    83                # gradient descent84                for layer in range(len(self.__network.getStructure())-1):85                    weights = self.__network.getConnectionWeights(layer)86                    weights -= (self.__learningRate / self.__miniBatchSize) * grads[layer] + self.__regularisationConst * weights87                    self.__network.setConnectionWeights(layer, weights)88                    89                    biases = self.__network.getNeuronBias(layer+1, range(self.__network.getStructure()[layer+1]))90                    biases -= (self.__learningRate / self.__miniBatchSize) * deltaSum[layer+1]91                    self.__network.setNeuronBias(layer+1, range(self.__network.getStructure()[layer+1]), biases)9293            lossHistoryTrainer.append(totalTrainingCost / self.__inputSize)94            95            # Validation (random subset of test set)96            np.random.shuffle(validationIndices)97            totalValidationCost = 098            totalErrors = 099            validationSetSize = max(int(self.__inputSize/60), 500)100            for validationIndex in range(validationSetSize):101                validationImage, actualLabel = self.__getValidationExample(validationIndices[validationIndex])102                testImageCost = self.__exampleCost(validationImage.reshape(784), actualLabel)103                totalValidationCost += testImageCost104                if actualLabel != np.argmax(self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))):105                    totalErrors += 1106            lossHistoryValidation.append(totalValidationCost / validationSetSize)107            errorHistory.append(10 * totalErrors / validationSetSize)108            109            self.__UIelements.drawGraphs(lossHistoryTrainer, lossHistoryValidation, errorHistory)110            self.__UIelements.writeToLog('done.\n')111            112            if 'saveIntervals' in kwargs and 'saveFileRoot' in kwargs:113                if kwargs['saveFileRoot'].endswith('.nn'):114                    fileRoot = kwargs['saveFileRoot'][:-len('.nn')]115                else:116                    fileRoot = kwargs['saveFileRoot']117                indexLength = int(np.ceil(np.log10(max(kwargs['saveIntervals'])+1)))118                if epoch+1 in kwargs['saveIntervals']:119                    saveFile = fileRoot + '{:0{}d}.nn'.format(epoch+1, indexLength)120                    nn.saveNetwork(self.__network, saveFile)121    122    def __exampleCost(self, inputLayer, desiredOutput):123        self.__network.setNeuronActivation(0, range(self.__network.getStructure()[0]), inputLayer)124        self.__network.evaluate()125        target = np.zeros(self.__network.getStructure()[-1])126        target[desiredOutput] = 1127        return np.sum((self.__network.getNeuronActivation(len(self.__network.getStructure())-1, range(self.__network.getStructure()[-1]))-target)**2)128    129    def setTrainingSet(self, trainingSetImages, trainingSetLabels):130        self.__trainingSetImages = trainingSetImages131        self.__trainingSetLabels = trainingSetLabels132        self.__trainingLoaded = True133        134    def __getTrainingExample(self, indices):135        images = self.__trainingSetImages[indices]136        labels = self.__trainingSetLabels[indices]137        return images, labels138    139    def setValidationSet(self, validationSetImages, validationSetLabels):140        self.__validationSetImages = validationSetImages141        self.__validationSetLabels = validationSetLabels
...

Compare.py

Source:Compare.py

...54        return mark, lost_tables_in_ref, lost_tables_in_tar55    def compTablesStructure(self, resTable, tagTable, sequential="1", col_properties="1"):56        if isinstance(resTable, Table) and isinstance(tagTable, Table):57            # è·ååååè¡¨58            resColumns = [structure[0] for structure in resTable.getStructure()]59            tagColumns = [structure[0] for structure in tagTable.getStructure()]60            list_msg = []61            mark = True62            # éªè¯ååä¸ªæ°63            if len(resColumns) != len(tagColumns):64                list_msg.append("table:%s columns:%d != table:%s columns:%d" % (resTable.getName(), len(resColumns),65                                                                                tagTable.getName(), len(tagColumns)))66            else:67                if sequential == "1":68                    for i in range(len(tagColumns)):69                        if tagColumns[i] != resColumns[i]:70                            list_msg.append("target:%s sequential != refence:%s sequential!" % (71                            tagTable.getName(), resTable.getName()))72            # è·åæ°å¢åæ®µä¸åå°åæ®µ73            columns_add = [column for column in tagColumns if column not in resColumns]74            columns_del = [column for column in resColumns if column not in tagColumns]75            if len(columns_add) > 0:76                list_msg.append("Added columns:%s" % ",".join(columns_add))77            if len(columns_del) > 0:78                list_msg.append("Deleted columns:%s" % ",".join(columns_del))79            # éªè¯åæ®µå±æ§80            if col_properties == "1":81                columns_both = [column for column in tagColumns if column in resColumns]82                dict_res = {structure[0]: structure for structure in resTable.getStructure() if83                            structure[0] in columns_both}84                dict_tag = {structure[0]: structure for structure in tagTable.getStructure() if85                            structure[0] in columns_both}86                for col in columns_both:87                    if dict_res[col] != dict_tag[col]:88                        list_msg.append("refrence column-%s:%s != target column-%s:%s" % (col, dict_res[col],89                                                                                          col, dict_tag[col]))90            if len(list_msg) > 0:91                mark = False92            msg = "\n".join(list_msg)93            return mark, msg94    def compareData(self, resTable, tagTable):95        if isinstance(resTable, Table) and isinstance(tagTable, Table):96            # å¼å§å¯¹æ¯æ°æ®97            list_msg = ["FAIL: %s != %s" % (resTable.getInfo(), tagTable.getInfo())]98            if len(resTable.getData()) != len(tagTable.getData()):99                list_msg.append("%s %d rows != %s %d rows" % (100                    resTable.getInfo(), len(resTable.getData()), tagTable.getInfo(), len(tagTable.getData())101                ))102            else:103                for line, resRow in enumerate(resTable.getData()):104                    if resRow not in tagTable.getData():  # è¯¥è¡ä¸å¨ç®æ è¡¨ä¸ï¼æ¤è¡æ°æ®å¯è½ä¸¢å¤±105                        if len(resTable.getPrimarykey()) > 0:  # ä¸»é®ä¸ä¸ºç©ºï¼è¦è¿è¡è¯¦ç»å¯¹æ¯106                            bool_row_status, str_rs = self.__compareRowData(resTable, tagTable, resRow)107                            if not bool_row_status:108                                list_msg.append(str_rs)109                            else:110                                list_msg.append("%s line %d is not found in %s\n(%s)" % (111                                    resTable.getInfo(), line + 1, tagTable.getInfo(),112                                    ",".join(map(lambda col: str(col), resRow))113                                ))114            if len(list_msg) == 1:115                print("*INFO* PASS: %s = %s " % (resTable.getInfo(), tagTable.getInfo()))116            msg = ("\n").join(list_msg)117            return len(list_msg) == 1, msg118    def __compareRowData(self, resTable, tagTable, resRow):119        list_resPri_index = [resTable.getStructure().index(key) for key in resTable.getPrimarykey()]120        list_tagPri_index = [tagTable.getStructure().index(key) for key in tagTable.getPrimarykey()]121        tag_index = []122        if not resTable.getView():123            tag_index = [tagTable.getStructure().index(resCol) for resCol in resTable.getStructure()]124        list_info = []125        for index in list_resPri_index:126            list_info.append("%s=%s" % (resTable.getStructure()[index], resRow[index]))127        line_info = " and ".join(list_info)128        errorMsg = ""129        find_row = False130        for tagRow in tagTable.getDb():131            flag = 0132            for i in xrange(len(list_resPri_index)):133                if resRow[list_resPri_index[i]] == tagRow[list_tagPri_index[i]]:134                    flag += 1135                else:136                    break137            if flag == len(list_resPri_index):  # è¯´ææ¾å°è¯¥è¡ï¼å¼å§å¯¹æ¯æ°æ®138                find_row = True139                list_msg = []140                if not resTable.getView():  # ä¸æ¯è§å¾ï¼ååå¨é¨åå141                    for i in xrange(len(tag_index)):142                        if resRow[i] != tagRow[tag_index[i]]:143                            str_msg = "%s(%s != %s)" % (resTable.getStructure()[i], resRow[i], tagRow[tag_index[i]])144                            list_msg.append(str_msg)145                else:  # è¡¨ä¸ºè§å¾ï¼ååå¯è½ä¸ä¸æ ·ï¼ä½æ¯å¯¹æ¯çæ°æ®ååºä¸ä¸å¯¹åº146                    for i in xrange(len(resRow)):147                        if resRow[i] != tagRow[i]:148                            str_msg = "%s.<%s> != %s.<%s>" % (149                                resTable.getStructure()[i], resRow[i], tagTable.getStructure()[i], tagRow[i])150                            list_msg.append(str_msg)151                if len(list_msg) > 0:152                    errorMsg = "Line %s    mismache columns:%s" % (line_info, "".join(list_msg))153                break  # æ¾å°è¿è¡ï¼éåºå¾ªç¯154        if not find_row:155            errorMsg = "Line %s is not found in %s" % (line_info, tagTable.getInfo())...

test_pipelines_asset.py

Source:test_pipelines_asset.py

...59def test_empty():60    fs = _get_test_fs()61    with mock_fs_scope(fs):62        expected = {}63        assert expected == fs.getStructure('counter')64        _bake_assets(fs)65        expected = {}66        assert expected == fs.getStructure('counter')67def test_one_file():68    fs = (_get_test_fs()69          .withFile('kitchen/assets/something.foo', 'A test file.'))70    with mock_fs_scope(fs):71        expected = {}72        assert expected == fs.getStructure('counter')73        _bake_assets(fs)74        expected = {'something.foo': 'A test file.'}75        assert expected == fs.getStructure('counter')76def test_one_level_dirtyness():77    fs = (_get_test_fs()78          .withFile('kitchen/assets/blah.foo', 'A test file.'))79    with mock_fs_scope(fs):80        _bake_assets(fs)81        expected = {'blah.foo': 'A test file.'}82        assert expected == fs.getStructure('counter')83        mtime = os.path.getmtime(fs.path('/counter/blah.foo'))84        assert abs(time.time() - mtime) <= 285        time.sleep(1)86        _bake_assets(fs)87        assert expected == fs.getStructure('counter')88        assert mtime == os.path.getmtime(fs.path('/counter/blah.foo'))89        time.sleep(1)90        fs.withFile('kitchen/assets/blah.foo', 'A new test file.')91        _bake_assets(fs)92        expected = {'blah.foo': 'A new test file.'}93        assert expected == fs.getStructure('counter')94        assert mtime < os.path.getmtime(fs.path('/counter/blah.foo'))95def test_two_levels_dirtyness():96    plugname = _get_test_plugin_name()97    fs = (_get_test_fs(plugins=[plugname], processors=['foo'])98          .withFile('kitchen/assets/blah.foo', 'A test file.'))99    _create_test_plugin(fs, plugname, foo_exts={'foo': 'bar'})100    with mock_fs_scope(fs):101        _bake_assets(fs)102        expected = {'blah.bar': 'FOO: A test file.'}103        assert expected == fs.getStructure('counter')104        mtime = os.path.getmtime(fs.path('/counter/blah.bar'))105        assert abs(time.time() - mtime) <= 2106        time.sleep(1)107        _bake_assets(fs)108        assert expected == fs.getStructure('counter')109        assert mtime == os.path.getmtime(fs.path('/counter/blah.bar'))110        time.sleep(1)111        fs.withFile('kitchen/assets/blah.foo', 'A new test file.')112        _bake_assets(fs)113        expected = {'blah.bar': 'FOO: A new test file.'}114        assert expected == fs.getStructure('counter')115        assert mtime < os.path.getmtime(fs.path('/counter/blah.bar'))116def test_removed():117    fs = (_get_test_fs()118          .withFile('kitchen/assets/blah1.foo', 'A test file.')119          .withFile('kitchen/assets/blah2.foo', 'Ooops'))120    with mock_fs_scope(fs):121        expected = {122            'blah1.foo': 'A test file.',123            'blah2.foo': 'Ooops'}124        assert expected == fs.getStructure('kitchen/assets')125        _bake_assets(fs)126        assert expected == fs.getStructure('counter')127        time.sleep(1)128        os.remove(fs.path('/kitchen/assets/blah2.foo'))129        expected = {130            'blah1.foo': 'A test file.'}131        assert expected == fs.getStructure('kitchen/assets')132        _bake_assets(fs)133        assert expected == fs.getStructure('counter')134def test_record_version_change():135    plugname = _get_test_plugin_name()136    fs = (_get_test_fs(plugins=[plugname], processors=['foo'])137          .withFile('kitchen/assets/blah.foo', 'A test file.'))138    _create_test_plugin(fs, plugname)139    with mock_fs_scope(fs):140        time.sleep(1)141        _bake_assets(fs)142        time.sleep(0.1)143        mtime = os.path.getmtime(fs.path('counter/blah.foo'))144        time.sleep(1)145        _bake_assets(fs)146        time.sleep(0.1)147        assert mtime == os.path.getmtime(fs.path('counter/blah.foo'))148        MultiRecord.RECORD_VERSION += 1149        try:150            time.sleep(1)151            _bake_assets(fs)152            time.sleep(0.1)153            assert mtime < os.path.getmtime(fs.path('counter/blah.foo'))154        finally:155            MultiRecord.RECORD_VERSION -= 1156@pytest.mark.parametrize('patterns, expected', [157    (['_'],158     {'something.html': 'A test file.'}),159    (['html'],160     {}),161    (['/^_/'],162     {'something.html': 'A test file.',163      'foo': {'_important.html': 'Important!'}})164])165def test_ignore_pattern(patterns, expected):166    fs = (_get_test_fs()167          .withFile('kitchen/assets/something.html', 'A test file.')168          .withFile('kitchen/assets/_hidden.html', 'Shhh')169          .withFile('kitchen/assets/foo/_important.html', 'Important!'))170    fs.withConfig({'pipelines': {'asset': {'ignore': patterns}}})171    with mock_fs_scope(fs):172        assert {} == fs.getStructure('counter')173        _bake_assets(fs)174        assert expected == fs.getStructure('counter')175@pytest.mark.parametrize('names, expected', [176    ('all', ['browserify', 'cleancss', 'compass', 'copy', 'concat', 'less',177             'requirejs', 'sass', 'sitemap', 'uglifyjs', 'pygments_style']),178    ('all -sitemap', ['browserify', 'cleancss', 'copy', 'compass', 'concat',179                      'less', 'requirejs', 'sass', 'uglifyjs',180                      'pygments_style']),181    ('-sitemap -less -sass all', ['browserify', 'cleancss', 'copy', 'compass',182                                  'concat', 'requirejs', 'uglifyjs',183                                  'pygments_style']),184    ('copy', ['copy']),185    ('less sass', ['less', 'sass'])186])187def test_filter_processor(names, expected):188    fs = mock_fs().withConfig()...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.