How to use context_manager method in localstack

Best Python code snippet using localstack_python

tests.py

Source:tests.py

1# Copyright 2017 Intel Corporation2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14# ------------------------------------------------------------------------------15# pylint: disable=too-many-lines,broad-except16import unittest17from collections import namedtuple18import hashlib19import os20import shutil21import tempfile22import time23from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase24from sawtooth_validator.execution import context_manager25from sawtooth_validator.state.merkle import MerkleDatabase26from sawtooth_validator.protobuf.events_pb2 import Event27TestAddresses = namedtuple('TestAddresses',28                           ['inputs', 'outputs', 'reads', 'writes'])29class TestContextManager(unittest.TestCase):30    def __init__(self, test_name):31        super().__init__(test_name)32        self._temp_dir = None33    def setUp(self):34        self._temp_dir = tempfile.mkdtemp()35        self.database_of_record = NativeLmdbDatabase(36            os.path.join(self._temp_dir, 'db_of_record.lmdb'),37            indexes=MerkleDatabase.create_index_configuration(),38            _size=10 * 1024 * 1024)39        self.context_manager = context_manager.ContextManager(40            self.database_of_record)41        self.first_state_hash = self.context_manager.get_first_root()42        # used for replicating state hash through direct merkle tree updates43        self.database_results = NativeLmdbDatabase(44            os.path.join(self._temp_dir, 'db_results.lmdb'),45            indexes=MerkleDatabase.create_index_configuration(),46            _size=10 * 1024 * 1024)47    def tearDown(self):48        self.context_manager.stop()49        shutil.rmtree(self._temp_dir)50    def _create_address(self, value=None):51        """52        Args:53            value: (str)54        Returns: (str) sha512 of value or random55        """56        if value is None:57            value = time.time().hex()58        return hashlib.sha512(value.encode()).hexdigest()[:70]59    def _setup_context(self):60        # 1) Create transaction data61        first_transaction = {'inputs': [self._create_address(a) for a in62                                        ['aaaa', 'bbbb', 'cccc']],63                             'outputs': [self._create_address(a) for a in64                                         ['llaa', 'aall', 'nnnn']]}65        second_transaction = {66            'inputs': [self._create_address(a) for a in67                       ['aaaa', 'dddd']],68            'outputs': [self._create_address(a) for a in69                        ['zzzz', 'yyyy', 'tttt', 'qqqq']]70        }71        third_transaction = {72            'inputs': [self._create_address(a) for a in73                       ['eeee', 'dddd', 'ffff']],74            'outputs': [self._create_address(a) for a in75                        ['oooo', 'oozz', 'zzoo', 'ppoo', 'aeio']]76        }77        # 2) Create contexts based on that data78        context_id_1 = self.context_manager.create_context(79            state_hash=self.first_state_hash,80            base_contexts=[],81            inputs=first_transaction['inputs'],82            outputs=first_transaction['outputs'])83        context_id_2 = self.context_manager.create_context(84            state_hash=self.first_state_hash,85            base_contexts=[],86            inputs=second_transaction['inputs'],87            outputs=second_transaction['outputs'])88        context_id_3 = self.context_manager.create_context(89            state_hash=self.first_state_hash,90            base_contexts=[],91            inputs=third_transaction['inputs'],92            outputs=third_transaction['outputs'])93        # 3) Set addresses with values94        self.context_manager.set(context_id_1, [{self._create_address(a): v}95                                                for a, v in [('llaa', b'1'),96                                                             ('aall', b'2'),97                                                             ('nnnn', b'3')]])98        self.context_manager.set(context_id_2, [{self._create_address(a): v}99                                                for a, v in [('zzzz', b'9'),100                                                             ('yyyy', b'11'),101                                                             ('tttt', b'12'),102                                                             ('qqqq', b'13')]])103        self.context_manager.set(context_id_3, [{self._create_address(a): v}104                                                for a, v in [('oooo', b'25'),105                                                             ('oozz', b'26'),106                                                             ('zzoo', b'27'),107                                                             ('ppoo', b'28'),108                                                             ('aeio', b'29')]])109        # 4)110        context_id = self.context_manager.create_context(111            state_hash=self.first_state_hash,112            base_contexts=[context_id_1, context_id_2, context_id_3],113            inputs=[114                self._create_address(a)115                for a in ['llaa', 'yyyy', 'tttt', 'zzoo']116            ],117            outputs=[118                self._create_address(a)119                for a in ['llaa', 'yyyy', 'tttt', 'zzoo', 'aeio']120            ])121        return context_id122    def _create_txn_inputs_outputs(self, start=None):123        """Create unique addresses that make up the inputs, outputs,124         reads, and writes that are involved in a context.125         Venn Diagram of relationship of disjoint sets that make up the126         inputs, outputs, reads, and writes.127         Knowledge of which disjoint set an address is a part of128         may give knowledge about a test failure in the context129         manager.130                    Inputs                      Outputs131            +----------+--------------------------+-----------+132            |          |                          |           |133            |  i___    |Reads       io__        Writes  _o__  |134            |          |                          |           |135            |    +-----------+-----------+---------------+    |136            |    |     |     |           |        |      |    |137            |    |     |     |           |        |      |    |138            |    |     |     |           |        |      |    |139            |    |     |     |           |        |      |    |140            |    |i_r_ | ior_|  iorw     |  io_w  | _o_w |    |141            |    |     |     |           |        |      |    |142            |    |     |     |           |        |      |    |143            |    |     |     |           |        |      |    |144            |    +-----------+-----------+---------------+    |145            |          |                          |           |146            |          |                          |           |147            +----------+--------------------------+-----------+148        Args:149            start (int): An integer to start the sequence of integers being150            hashed to addresses.151        Returns (namedtuple): An object that holds inputs, outputs, reads,152            and writes.153        """154        if start is None:155            start = 0156        iorw = [self._create_address(str(i)) for i in range(start, start + 10)]157        i_r_ = [158            self._create_address(str(i))159            for i in range(start + 10, start + 20)160        ]161        ior_ = [162            self._create_address(str(i))163            for i in range(start + 20, start + 30)164        ]165        io__ = [166            self._create_address(str(i))167            for i in range(start + 30, start + 40)168        ]169        io_w = [170            self._create_address(str(i))171            for i in range(start + 40, start + 50)172        ]173        _o_w = [174            self._create_address(str(i))175            for i in range(start + 50, start + 60)176        ]177        _o__ = [178            self._create_address(str(i))179            for i in range(start + 60, start + 70)180        ]181        i___ = [182            self._create_address(str(i))183            for i in range(start + 70, start + 80)184        ]185        addresses = TestAddresses(186            inputs=iorw + ior_ + io__ + io_w + i___,187            outputs=ior_ + io__ + io_w + _o__ + _o_w,188            reads=i_r_ + ior_,189            writes=io_w + _o_w)190        return addresses191    def test_execution_results(self):192        """Tests that get_execution_results returns the correct values."""193        addr1 = self._create_address()194        addr2 = self._create_address()195        context_id = self.context_manager.create_context(196            state_hash=self.context_manager.get_first_root(),197            base_contexts=[],198            inputs=[addr1, addr2],199            outputs=[addr1, addr2])200        sets = {addr1: b'1'}201        events = [202            Event(203                event_type=teststr,204                attributes=[Event.Attribute(key=teststr, value=teststr)],205                data=teststr.encode()) for teststr in ("test1", "test2")206        ]207        deletes = {addr2: None}208        data = [(teststr.encode()) for teststr in ("test1", "test2")]209        self.context_manager.set(context_id, [sets])210        for event in events:211            self.context_manager.add_execution_event(context_id, event)212        self.context_manager.delete(context_id, deletes)213        for datum in data:214            self.context_manager.add_execution_data(215                context_id, datum)216        results = self.context_manager.get_execution_results(context_id)217        self.assertEqual(sets, results[0])218        self.assertEqual(deletes, results[1])219        self.assertEqual(events, results[2])220        self.assertEqual(data, results[3])221    def test_address_enforcement(self):222        """Tests that the ContextManager enforces address characteristics.223        Notes:224            1. Call get and set on the ContextManager with an address that is225               under a namespace, but is an invalid address, and test that226               the methods raise an AuthorizationException.227        """228        # 1)229        invalid_address1 = 'a' * 69 + 'n'230        invalid_address2 = 'b' * 69 + 'y'231        context_id1 = self.context_manager.create_context(232            state_hash=self.context_manager.get_first_root(),233            base_contexts=[],234            inputs=['aaaaaaaa', 'bbbbbbbb'],235            outputs=['aaaaaaaa', 'bbbbbbbb'])236        with self.assertRaises(context_manager.AuthorizationException):237            self.context_manager.get(238                context_id=context_id1,239                address_list=[invalid_address1, invalid_address2])240        with self.assertRaises(context_manager.AuthorizationException):241            self.context_manager.set(242                context_id=context_id1,243                address_value_list=[{invalid_address1: b'1'},244                                    {invalid_address2: b'2'}])245    def test_get_set_wrong_namespace(self):246        """Tests that getting and setting from outside the namespace will247        raise a AuthorizationException.248        Notes:249            1. Assert that sets on a context with addresses that aren't250               under an output namespace raise an AuthorizationException.251            2. Assert that gets on a context with addresses that aren't under252               an input namespace raise an AuthorizationException.253        """254        wrong_namespace1 = self._create_address('a')[-10:]255        wrong_namespace2 = '00000000'256        ctx_1 = self.context_manager.create_context(257            state_hash=self.context_manager.get_first_root(),258            base_contexts=[],259            inputs=[wrong_namespace1, wrong_namespace2],260            outputs=[wrong_namespace1, wrong_namespace2])261        # 1262        with self.assertRaises(context_manager.AuthorizationException):263            self.context_manager.set(264                context_id=ctx_1,265                address_value_list=[{self._create_address('a'): b'1'}])266        with self.assertRaises(context_manager.AuthorizationException):267            self.context_manager.set(268                context_id=ctx_1,269                address_value_list=[{self._create_address('c'): b'5'}])270        # 2271        with self.assertRaises(context_manager.AuthorizationException):272            self.context_manager.get(273                context_id=ctx_1,274                address_list=[self._create_address('a')])275        with self.assertRaises(context_manager.AuthorizationException):276            self.context_manager.get(277                context_id=ctx_1,278                address_list=[self._create_address('c')])279    def test_exception_on_invalid_input(self):280        """Tests that invalid inputs raise an exception. Tested with invalid281        characters, odd number of characters, and too long namespace;282        Notes:283            1) Assert that inputs with a namespace with an odd number of284               characters raise a CreateContextException.285            2) Assert that inputs with a 71 character namespace raise a286               CreateContextException.287            3) Assert that inputs with a namespace with several invalid288               characters raise a CreateContextException.289        """290        invalid_input_output1 = '0db7e8zc'  # invalid character291        invalid_input_output2 = '7ef84ed' * 10 + '5'  # too long, 71 chars292        invalid_input_output3 = 'yy76ftoph7465873ddde389f'  # invalid chars293        valid_input_output1 = 'd8f533bbb74443222daad4'294        valid_input_output2 = '77465847465784757848ddddddf'295        state_hash = self.context_manager.get_first_root()296        # 1297        with self.assertRaises(context_manager.CreateContextException):298            self.context_manager.create_context(299                state_hash=state_hash,300                base_contexts=[],301                inputs=[invalid_input_output1, valid_input_output1],302                outputs=[valid_input_output2])303        # 2304        with self.assertRaises(context_manager.CreateContextException):305            self.context_manager.create_context(306                state_hash=state_hash,307                base_contexts=[],308                inputs=[valid_input_output1, invalid_input_output2],309                outputs=[valid_input_output2])310        # 3311        with self.assertRaises(context_manager.CreateContextException):312            self.context_manager.create_context(313                state_hash=state_hash,314                base_contexts=[],315                inputs=[invalid_input_output3, valid_input_output2],316                outputs=[valid_input_output2, valid_input_output1])317    def test_exception_on_invalid_output(self):318        """Tests that invalid outputs raise an exception. Tested with invalid319        characters, odd number of characters, and too long namespace;320        Notes:321            1) Assert that outputs with a namespace with an odd number of322               characters raise a CreateContextException.323            2) Assert that outputs with a 71 character namespace raise a324               CreateContextException.325            3) Assert that outputs with a namespace with several invalid326               characters raise a CreateContextException.327        """328        invalid_input_output1 = '0db7e87'  # Odd number of characters329        invalid_input_output2 = '7ef84ed' * 10 + '5'  # too long, 71 chars330        invalid_input_output3 = 'yy76ftoph7465873ddde389f'  # invalid chars331        valid_input_output1 = 'd8f533bbb74443222daad4'332        valid_input_output2 = '77465847465784757848ddddddff'333        state_hash = self.context_manager.get_first_root()334        # 1335        with self.assertRaises(context_manager.CreateContextException):336            self.context_manager.create_context(337                state_hash=state_hash,338                base_contexts=[],339                inputs=[valid_input_output2, valid_input_output1],340                outputs=[invalid_input_output1])341        # 2342        with self.assertRaises(context_manager.CreateContextException):343            self.context_manager.create_context(344                state_hash=state_hash,345                base_contexts=[],346                inputs=[valid_input_output1, valid_input_output2],347                outputs=[invalid_input_output2])348        # 3349        with self.assertRaises(context_manager.CreateContextException):350            self.context_manager.create_context(351                state_hash=state_hash,352                base_contexts=[],353                inputs=[valid_input_output1, valid_input_output2],354                outputs=[valid_input_output2, invalid_input_output3])355    def test_namespace_gets(self):356        """Tests that gets for an address under a namespace will return the357        correct value.358        Notes:359            1) Create ctx_1 and set 'b' to b'8'.360            2) squash the previous context creating state_hash_1.361            3) Create 2 contexts off of this state hash and assert362               that gets on these contexts retrieve the correct363               value for an address that is not fully specified in the inputs.364            4) Set values to addresses in these contexts.365            5) Create 1 context off of these prior 2 contexts and assert that366               gets from this context retrieve the correct values for367               addresses that are not fully specified in the inputs. 2 of the368               values are found in the chain of contexts, and 1 is not found369               and so must be retrieved from the merkle tree.370        """371        # 1372        ctx_1 = self.context_manager.create_context(373            state_hash=self.context_manager.get_first_root(),374            base_contexts=[],375            inputs=[self._create_address('a')],376            outputs=[self._create_address('b')])377        self.context_manager.set(378            context_id=ctx_1,379            address_value_list=[{self._create_address('b'): b'8'}])380        # 2381        squash = self.context_manager.get_squash_handler()382        state_hash_1 = squash(383            state_root=self.context_manager.get_first_root(),384            context_ids=[ctx_1],385            persist=True,386            clean_up=True)387        # 3388        ctx_1a = self.context_manager.create_context(389            state_hash=state_hash_1,390            base_contexts=[],391            inputs=[self._create_address('a')[:10]],392            outputs=[self._create_address('c')])393        self.assertEqual(394            self.context_manager.get(395                context_id=ctx_1a,396                address_list=[self._create_address('a')]),397            [(self._create_address('a'), None)])398        ctx_1b = self.context_manager.create_context(399            state_hash=state_hash_1,400            base_contexts=[],401            inputs=[self._create_address('b')[:6]],402            outputs=[self._create_address('z')])403        self.assertEqual(404            self.context_manager.get(405                context_id=ctx_1b,406                address_list=[self._create_address('b')]),407            [(self._create_address('b'), b'8')])408        # 4409        self.context_manager.set(410            context_id=ctx_1b,411            address_value_list=[{self._create_address('z'): b'2'}])412        self.context_manager.set(413            context_id=ctx_1a,414            address_value_list=[{self._create_address('c'): b'1'}]415        )416        ctx_2 = self.context_manager.create_context(417            state_hash=state_hash_1,418            base_contexts=[ctx_1a, ctx_1b],419            inputs=[420                self._create_address('z')[:10],421                self._create_address('c')[:10],422                self._create_address('b')[:10]423            ],424            outputs=[self._create_address('w')])425        self.assertEqual(426            self.context_manager.get(427                context_id=ctx_2,428                address_list=[self._create_address('z'),429                              self._create_address('c'),430                              self._create_address('b')]),431            [(self._create_address('z'), b'2'),432             (self._create_address('c'), b'1'),433             (self._create_address('b'), b'8')])434    def test_create_context_with_prior_state(self):435        """Tests context creation with prior state from base contexts.436        Notes:437            Set up the context:438                Create 3 prior contexts each with 3-5 addresses to set to.439                Make set calls to those addresses.440                Create 1 new context based on those three prior contexts.441                this test method:442            Test:443                Make a get call on addresses that are from prior state,444                making assertions about the correct values.445        """446        context_id = self._setup_context()447        self.assertEqual(self.context_manager.get(448            context_id,449            [self._create_address(a) for a in450             ['llaa', 'yyyy', 'tttt', 'zzoo']]),451            [(self._create_address(a), v) for a, v in452             [('llaa', b'1'),453              ('yyyy', b'11'),454              ('tttt', b'12'),455              ('zzoo', b'27')]])456    def test_squash(self):457        """Tests that squashing a context based on state from other458        contexts will result in the same merkle hash as updating the459        merkle tree with the same data.460        Notes:461            Set up the context462            Test:463                1) Make set calls on several of the addresses.464                2) Squash the context to get a new state hash.465                3) Apply all of the aggregate sets from all466                of the contexts, to another database with a merkle tree.467                4) Assert that the state hashes are the same.468        """469        # 1)470        context_id = self._setup_context()471        self.context_manager.set(472            context_id,473            [{self._create_address(a): v} for a, v in474             [('yyyy', b'2'),475              ('tttt', b'4')]])476        # 2)477        squash = self.context_manager.get_squash_handler()478        resulting_state_hash = squash(self.first_state_hash, [context_id],479                                      persist=True, clean_up=True)480        # 3)481        final_state_to_update = {self._create_address(a): v for a, v in482                                 [('llaa', b'1'),483                                  ('aall', b'2'),484                                  ('nnnn', b'3'),485                                  ('zzzz', b'9'),486                                  ('yyyy', b'2'),487                                  ('tttt', b'4'),488                                  ('qqqq', b'13'),489                                  ('oooo', b'25'),490                                  ('oozz', b'26'),491                                  ('zzoo', b'27'),492                                  ('ppoo', b'28'),493                                  ('aeio', b'29')]}494        test_merkle_tree = MerkleDatabase(self.database_results)495        test_resulting_state_hash = test_merkle_tree.update(496            final_state_to_update, virtual=False)497        # 4)498        self.assertEqual(resulting_state_hash, test_resulting_state_hash)499    def test_squash_no_updates(self):500        """Tests that squashing a context that has no state updates will return501           the starting state root hash.502        Notes:503            Set up the context504            Test:505                1) Squash the context.506                2) Assert that the state hash is the same as the starting507                hash.508        """509        context_id = self.context_manager.create_context(510            state_hash=self.first_state_hash,511            base_contexts=[],512            inputs=[],513            outputs=[])514        # 1)515        squash = self.context_manager.get_squash_handler()516        resulting_state_hash = squash(self.first_state_hash, [context_id],517                                      persist=True, clean_up=True)518        # 2519        self.assertIsNotNone(resulting_state_hash)520        self.assertEqual(resulting_state_hash, self.first_state_hash)521    def test_squash_deletes_no_update(self):522        """Tests that squashing a context that has no state updates,523        due to sets that were subsequently deleted, will return524        the starting state root hash.525        Notes:526            Set up the context527            Test:528                1) Send Updates that reverse each other.529                2) Squash the context.530                3) Assert that the state hash is the same as the starting531                hash.532        """533        context_id = self.context_manager.create_context(534            state_hash=self.first_state_hash,535            base_contexts=[],536            inputs=[],537            outputs=[self._create_address(a) for a in538                     ['yyyy', 'tttt']])539        # 1)540        self.context_manager.set(541            context_id,542            [{self._create_address(a): v} for a, v in543             [('yyyy', b'2'),544              ('tttt', b'4')]])545        self.context_manager.delete(546            context_id,547            [self._create_address(a) for a in548             ['yyyy', 'tttt']])549        # 2)550        squash = self.context_manager.get_squash_handler()551        resulting_state_hash = squash(self.first_state_hash, [context_id],552                                      persist=True, clean_up=True)553        # 3)554        self.assertIsNotNone(resulting_state_hash)555        self.assertEqual(resulting_state_hash, self.first_state_hash)556    def test_reads_from_context_w_several_writes(self):557        """Tests that those context values that have been written to the558        Merkle tree, or that have been set to a base_context, will have the559        correct value at the address for a given context.560                                               ->context_id_a1561                                               |              |562                                               |              |563                                               |              |564        sh0-->context_id1-->sh1-->context_a-----              -->context_id_b565                                               |              |566                                               |              |567                                               |              |568                                               |              |569                                               -->context_id_a2570        Notes:571            Test:572                1. From a Merkle Tree with only the root node in it, create a573                   context and set several values, and then squash that context574                   upon the first state hash.575                2. Create a context with no base context, based on576                   the merkle root computed from the first squash.577                   Assert that gets from this context will provide578                   values that were set in the first context.579                3. Write to all of the available outputs580                4. Create a new context based on context_a, from #2,581                5. Assert that gets from this context equal the values set582                   to Context A.583                6. Create a new context based on context_a and set values to584                   this context.585                7. Create a new context based on the 2 contexts made in 4 and 6586                8. From this context assert that gets equal the correct values587                   set in the prior contexts.588        """589        squash = self.context_manager.get_squash_handler()590        test_addresses = self._create_txn_inputs_outputs()591        # 1)592        context_id1 = self.context_manager.create_context(593            state_hash=self.first_state_hash,594            inputs=test_addresses.inputs,595            outputs=test_addresses.outputs,596            base_contexts=[])597        values1 = [bytes(i) for i in range(len(test_addresses.writes))]598        self.context_manager.set(599            context_id1,600            [{a: v} for a, v in zip(test_addresses.writes, values1)])601        sh1 = squash(602            state_root=self.first_state_hash,603            context_ids=[context_id1],604            persist=True,605            clean_up=True)606        # 2)607        context_a = self.context_manager.create_context(608            state_hash=sh1,609            inputs=test_addresses.writes,  # read from every address written to610            outputs=test_addresses.outputs,611            base_contexts=[]612        )613        address_values = self.context_manager.get(614            context_a,615            list(test_addresses.writes)616        )617        self.assertEqual(618            address_values,619            list(zip(test_addresses.writes, values1))620        )621        # 3)622        values2 = [bytes(v.encode()) for v in test_addresses.outputs]623        self.context_manager.set(624            context_id=context_a,625            address_value_list=[dict(zip(test_addresses.outputs, values2))])626        # 4)627        context_id_a1 = self.context_manager.create_context(628            state_hash=sh1,629            inputs=test_addresses.outputs,630            outputs=test_addresses.outputs,631            base_contexts=[context_a]632        )633        # 5)634        c_ida1_address_values = self.context_manager.get(635            context_id=context_id_a1,636            address_list=list(test_addresses.outputs)637        )638        self.assertEqual(639            c_ida1_address_values,640            list(zip(test_addresses.outputs, values2))641        )642        # 6)643        test_addresses2 = self._create_txn_inputs_outputs(80)644        context_id_a2 = self.context_manager.create_context(645            state_hash=sh1,646            inputs=test_addresses2.inputs,647            outputs=test_addresses2.outputs,648            base_contexts=[context_a]649        )650        values3 = [bytes(v.encode()) for v in test_addresses2.writes]651        self.context_manager.set(652            context_id=context_id_a2,653            address_value_list=[{a: v} for654                                a, v in zip(test_addresses2.writes, values3)],655        )656        # 7)657        context_id_b = self.context_manager.create_context(658            state_hash=sh1,659            inputs=test_addresses2.writes + test_addresses.outputs,660            outputs=[],661            base_contexts=[context_id_a1, context_id_a2]662        )663        # 8)664        self.assertEqual(665            self.context_manager.get(666                context_id_b,667                list(test_addresses2.writes + test_addresses.outputs)668            ),669            list(zip(670                test_addresses2.writes + test_addresses.outputs,671                values3 + values2))672        )673    def test_state_root_after_parallel_ctx(self):674        """Tests that the correct state root is calculated after basing one675        context off of multiple contexts.676                              i=abcd677                              o=aaaa678                           +>context_1+679                           |  aaaa=1  |680                           |          |681               i=llll      |   i=bacd |      i=bbbb,aaaa682               o=llll      |   o=bbbb |      o=cccc,llll683        sh0--->ctx_0-->sh1>|-->context_2-+---->context_n---->sh2684               llll=5      |   bbbb=2 |      cccc=4685                           |          |      llll=8686                           |   i=abcd |687                           |   o=cccc |688                           +>context_3+689                               cccc=3690        Notes:691            Test:692                1. Create a context, set a value in it and squash it into a new693                   state hash.694                2. Create 3 contexts based off of the state root from #1.695                3. Set values at addresses to all three contexts.696                4. Base another context off of the contexts from #2.697                5. Set a value to an address in this context that has already698                   been set to in the non-base context.699                6. Squash the contexts producing a state hash and assert700                   that it equals a state hash obtained by manually updating701                   the merkle tree.702        """703        sh0 = self.first_state_hash704        # 1)705        squash = self.context_manager.get_squash_handler()706        ctx_1 = self.context_manager.create_context(707            state_hash=sh0,708            base_contexts=[],709            inputs=[self._create_address('llll')],710            outputs=[self._create_address('llll')]711        )712        self.context_manager.set(713            context_id=ctx_1,714            address_value_list=[{self._create_address('llll'): b'5'}]715        )716        sh1 = squash(717            state_root=sh0,718            context_ids=[ctx_1],719            persist=True,720            clean_up=True)721        # 2)722        context_1 = self.context_manager.create_context(723            state_hash=sh1,724            base_contexts=[],725            inputs=[self._create_address('abcd')],726            outputs=[self._create_address('aaaa')])727        context_2 = self.context_manager.create_context(728            state_hash=sh1,729            base_contexts=[],730            inputs=[self._create_address('bacd')],731            outputs=[self._create_address('bbbb')])732        context_3 = self.context_manager.create_context(733            state_hash=sh1,734            base_contexts=[],735            inputs=[self._create_address('abcd')],736            outputs=[737                self._create_address('cccc'),738                self._create_address('dddd')739            ])740        # 3)741        self.context_manager.set(742            context_id=context_1,743            address_value_list=[{self._create_address('aaaa'): b'1'}]744        )745        self.context_manager.set(746            context_id=context_2,747            address_value_list=[{self._create_address('bbbb'): b'2'}]748        )749        self.context_manager.set(750            context_id=context_3,751            address_value_list=[{self._create_address('cccc'): b'3'}]752        )753        # 4)754        context_n = self.context_manager.create_context(755            state_hash=sh1,756            base_contexts=[context_1, context_2, context_3],757            inputs=[758                self._create_address('bbbb'),759                self._create_address('aaaa')760            ],761            outputs=[762                self._create_address('cccc'),763                self._create_address('llll')764            ])765        # 5)766        self.context_manager.set(767            context_id=context_n,768            address_value_list=[{769                self._create_address('cccc'): b'4',770                self._create_address('llll'): b'8'771            }])772        # 6)773        cm_state_root = squash(774            state_root=sh1,775            context_ids=[context_n],776            persist=False,777            clean_up=True)778        tree = MerkleDatabase(self.database_results)779        calc_state_root = tree.update({780            self._create_address('aaaa'): b'1',781            self._create_address('bbbb'): b'2',782            self._create_address('cccc'): b'4',783            self._create_address('llll'): b'8'784        })785        self.assertEqual(calc_state_root, cm_state_root)786    def test_complex_basecontext_squash(self):787        """Tests complex context basing and squashing.788                                            i=qq,dd dd=0789                                            o=dd,pp pp=1790                                i=cc,aa  +->context_3_2a_1+|791                                o=dd,ll  |                 |792               i=aa,ab      +->context_2a|  i=aa    aa=0   |793               o=cc,ab      |   dd=10    |  o=aa,ll ll=1   |794        sh0->context_1-->sh1|   ll=11    +->context_3_2a_2+|->sh1795               cc=0         |   i=cc,aa  +->context_3_2b_1+|796               ab=1         |   o=nn,mm  |  i=nn,ba mm=0   |797                            +->context_2b|  o=mm,ba ba=1   |798                                nn=0     |                 |799                                mm=1     +->context_3_2b_2+|800                                            i=nn,oo ab=0801                                            o=ab,oo oo=1802        Notes:803            Test:804                1. Create a context off of the first state hash, set805                   addresses in it, and squash that context, getting a new806                   merkle root.807                2. Create 2 contexts with the context in # 1 as the base, and808                   for each of these contexts set addresses to values where the809                   outputs for each are disjoint.810                3. For each of these 2 contexts create 2 more contexts each811                   having one of the contexts in # 2 as the base context, and812                   set addresses to values.813                4. Squash the 4 contexts from #3 and assert the state hash814                   is equal to a manually computed state hash.815        """816        squash = self.context_manager.get_squash_handler()817        # 1)818        inputs_1 = [self._create_address('aa'),819                    self._create_address('ab')]820        outputs_1 = [self._create_address('cc'),821                     self._create_address('ab')]822        context_1 = self.context_manager.create_context(823            state_hash=self.first_state_hash,824            base_contexts=[],825            inputs=inputs_1,826            outputs=outputs_1)827        self.context_manager.set(828            context_id=context_1,829            address_value_list=[{a: v} for a, v in zip(830                outputs_1, [bytes(i) for i in range(len(outputs_1))])])831        sh1 = squash(832            state_root=self.first_state_hash,833            context_ids=[context_1],834            persist=True,835            clean_up=True)836        # 2)837        inputs_2a = [self._create_address('cc'),838                     self._create_address('aa')]839        outputs_2a = [self._create_address('dd'),840                      self._create_address('ll')]841        context_2a = self.context_manager.create_context(842            state_hash=self.first_state_hash,843            base_contexts=[],844            inputs=inputs_2a,845            outputs=outputs_2a)846        inputs_2b = [self._create_address('cc'),847                     self._create_address('aa')]848        outputs_2b = [self._create_address('nn'),849                      self._create_address('mm')]850        context_2b = self.context_manager.create_context(851            state_hash=sh1,852            base_contexts=[],853            inputs=inputs_2b,854            outputs=outputs_2b)855        self.context_manager.set(856            context_id=context_2a,857            address_value_list=[{a: bytes(v)}858                                for a, v in zip(outputs_2a,859                                                range(10,860                                                      10 + len(outputs_2a)))]861        )862        self.context_manager.set(863            context_id=context_2b,864            address_value_list=[{a: bytes(v)}865                                for a, v in zip(outputs_2b,866                                                range(len(outputs_2b)))]867        )868        # 3)869        inputs_3_2a_1 = [870            self._create_address('qq'),871            self._create_address('dd')872        ]873        outputs_3_2a_1 = [874            self._create_address('dd'),875            self._create_address('pp')876        ]877        context_3_2a_1 = self.context_manager.create_context(878            state_hash=sh1,879            base_contexts=[context_2a],880            inputs=inputs_3_2a_1,881            outputs=outputs_3_2a_1)882        inputs_3_2a_2 = [self._create_address('aa')]883        outputs_3_2a_2 = [884            self._create_address('aa'),885            self._create_address('ll')886        ]887        context_3_2a_2 = self.context_manager.create_context(888            state_hash=sh1,889            base_contexts=[context_2a],890            inputs=inputs_3_2a_2,891            outputs=outputs_3_2a_2)892        inputs_3_2b_1 = [893            self._create_address('nn'),894            self._create_address('ab')895        ]896        outputs_3_2b_1 = [897            self._create_address('mm'),898            self._create_address('ba')899        ]900        context_3_2b_1 = self.context_manager.create_context(901            state_hash=sh1,902            base_contexts=[context_2b],903            inputs=inputs_3_2b_1,904            outputs=outputs_3_2b_1)905        inputs_3_2b_2 = [906            self._create_address('nn'),907            self._create_address('oo')908        ]909        outputs_3_2b_2 = [910            self._create_address('ab'),911            self._create_address('oo')912        ]913        context_3_2b_2 = self.context_manager.create_context(914            state_hash=sh1,915            base_contexts=[context_2b],916            inputs=inputs_3_2b_2,917            outputs=outputs_3_2b_2)918        self.context_manager.set(919            context_id=context_3_2a_1,920            address_value_list=[{a: bytes(v)}921                                for a, v in zip(outputs_3_2a_1,922                                                range(len(outputs_3_2a_1)))])923        self.context_manager.set(924            context_id=context_3_2a_2,925            address_value_list=[{a: bytes(v)}926                                for a, v in zip(outputs_3_2a_2,927                                                range(len(outputs_3_2a_2)))])928        self.context_manager.set(929            context_id=context_3_2b_1,930            address_value_list=[{a: bytes(v)}931                                for a, v in zip(outputs_3_2b_1,932                                                range(len(outputs_3_2b_1)))])933        self.context_manager.set(934            context_id=context_3_2b_2,935            address_value_list=[{a: bytes(v)}936                                for a, v in zip(outputs_3_2b_2,937                                                range(len(outputs_3_2b_2)))])938        # 4)939        sh2 = squash(940            state_root=sh1,941            context_ids=[context_3_2a_1, context_3_2a_2,942                         context_3_2b_1, context_3_2b_2],943            persist=False,944            clean_up=True)945        tree = MerkleDatabase(self.database_results)946        state_hash_from_1 = tree.update(947            set_items=dict(zip(outputs_1,948                               [bytes(i) for i in range(len(outputs_1))])),949            virtual=False)950        self.assertEqual(state_hash_from_1, sh1,951                         "The manually calculated state hash from the first "952                         "context and the one calculated by squashing that "953                         "state hash should be the same")954        tree.set_merkle_root(state_hash_from_1)955        test_sh2 = tree.update(956            set_items={957                self._create_address('aa'): bytes(0),958                self._create_address('ab'): bytes(0),959                self._create_address('ba'): bytes(1),960                self._create_address('dd'): bytes(0),961                self._create_address('ll'): bytes(1),962                self._create_address('mm'): bytes(0),963                self._create_address('oo'): bytes(1),964                self._create_address('pp'): bytes(1),965                self._create_address('nn'): bytes(0),966                self._create_address('cc'): bytes(0)})967        self.assertEqual(sh2, test_sh2, "Manually calculated and context "968                         "manager calculated merkle hashes "969                         "are the same")970    def test_wildcarded_inputs_outputs(self):971        """Tests the context manager with wildcarded inputs and outputs.972        Notes:973            1. Create a context with a wildcarded input and output and974               another non-wildcarded input and output.975            2. Get an address under the wildcard and set to both the976               non-wildcarded address and an address under the977               wildcard.978            3. Squash the context and compare to a manually generated979               state hash.980        """981        # 1982        namespaces = [983            self._create_address('a')[:8],984            self._create_address('b')985        ]986        ctx_1 = self.context_manager.create_context(987            inputs=namespaces,988            outputs=namespaces,989            base_contexts=[],990            state_hash=self.first_state_hash)991        # 2992        self.context_manager.get(993            context_id=ctx_1,994            address_list=[self._create_address('a')])995        self.context_manager.set(996            context_id=ctx_1,997            address_value_list=[{998                self._create_address('a'): b'1',999                self._create_address('b'): b'2'1000            }])1001        # 31002        squash = self.context_manager.get_squash_handler()1003        tree = MerkleDatabase(self.database_results)1004        tree.set_merkle_root(self.first_state_hash)1005        calculated_state_root = tree.update(1006            set_items={self._create_address('a'): b'1',1007                       self._create_address('b'): b'2'})1008        state_root = squash(1009            state_root=self.first_state_hash,1010            context_ids=[ctx_1],1011            persist=True,1012            clean_up=True)1013        self.assertEqual(state_root, calculated_state_root)1014    @unittest.skip("Necessary to catch scheduler bugs--Depth-first search")1015    def test_check_for_bad_combination(self):1016        """Tests that the context manager will raise1017        an exception if asked to combine contexts, either via base contexts1018        in create_context or via squash that shouldn't be1019        combined because they share addresses that can't be determined by the1020        scheduler to not have been parallel. This is a check on scheduler bugs.1021        Examples where the context manager should raise an exception on1022        duplicate addresses:1023        1. Success1024              i=a1025              o=b1026           +>ctx_1+1027        dup|->b=3 |1028           |      |1029        sh0|      ----->state hash or context1030           |  i=q |1031           |  o=b |1032           +>ctx_2+1033        dup-->b=21034        2.1035                      i=b1036                      o=d1037                   +>ctx_1a_1+1038                   |  d=4    |1039             i=a   |         |1040             o=b   |         |1041           +>ctx_1a|         |1042           | b=2   |  i=b    |1043           |       |  o=c    |1044        sh0|       +>ctx_1a_2|1045           |     dup-> c=7   |------>state hash or context1046           |  i=a  +>ctx_1b_1|1047           |  o=c  |         |1048           +>ctx_1b|         |1049        dup-->c=5  |  i=t    |1050                   |  o=p    |1051                   +>ctx_1b_2+1052                      p=81053        3.1054                      i=b1055                      o=d1056                   +>ctx_1a_1+1057                   |  d=4    |   i=d,c1058             i=a   |         |   o=n1059             o=b   |         <>ctx_3a+1060           +>ctx_1a|         |   n=5 |1061           | b=2   |  i=b    |       |1062           |       |  o=c    |       |1063        sh0|       +>ctx_1a_2+       <----->state hash or context1064           |   dup--> c=7            |1065           |  i=a  +>ctx_1b_1+       |1066           |  o=c  |         |  i=c  |1067           +>ctx_1b|         |  o=q  |1068              c=5  |  i=c    <>ctx_3b+1069                   |  o=c    |  q=51070                   +>ctx_1b_2+1071               dup--> c=11072        """1073        # 1.1074        squash = self.context_manager.get_squash_handler()1075        sh0 = self.first_state_hash1076        inputs_1 = [self._create_address('a')]1077        outputs_1 = [self._create_address('b')]1078        ctx_1 = self.context_manager.create_context(1079            state_hash=sh0,1080            base_contexts=[],1081            inputs=inputs_1,1082            outputs=outputs_11083        )1084        self.context_manager.set(1085            context_id=ctx_1,1086            address_value_list=[{self._create_address('b'): b'3'}]1087        )1088        inputs_2 = [self._create_address('q')]1089        outputs_2 = [self._create_address('b')]1090        ctx_2 = self.context_manager.create_context(1091            state_hash=sh0,1092            base_contexts=[],1093            inputs=inputs_2,1094            outputs=outputs_2)1095        self.context_manager.set(1096            context_id=ctx_2,1097            address_value_list=[{1098                self._create_address('b'): b'2'1099            }])1100        try:1101            squash(1102                state_root=sh0,1103                context_ids=[ctx_1, ctx_2],1104                persist=True,1105                clean_up=True)1106            self.fail("squash of two contexts with a duplicate address")1107        except Exception:1108            pass1109    def test_simple_read_write_delete(self):1110        """Tests that after sets and deletes, subsequent contexts have the1111        correct value in the context for an address.1112                       i:a1113                       o:b1114                   +-->ctx_2a+1115                   |   s:b:2 |1116                   |         |1117              i:a  |         |    i:b        i:a,b,c1118              o:a  |         |    o:b,c      o:c1119        sh0+->ctx_1|         <--->ctx_3+---->ctx_4+------>sh11120              s:a:1|         |    d:b,c      s:c:41121                   |         |1122                   |   i:a   |1123                   |   o:c   |1124                   +-->ctx_2b+1125                       s:c:21126        Notes:1127            1. From the diagram:1128                - ctx_1, with input 'a' and output 'a' will have 'a' set to1129                  b'1'.1130                - ctx_2a will have 'b' set to b'2'1131                - ctx_2b will have 'c' set to b'2'1132                - ctx_3 will delete both 'b' and 'c'1133                - ctx_4 will set 'c' to b'4'1134                - ctx_4 will be squashed along with it's base contexts into1135                  a new state hash.1136            2. Assertions1137                - Assert for every context that it has the correct state1138                  before the set or delete.1139        """1140        sh0 = self.context_manager.get_first_root()1141        ctx_1 = self.context_manager.create_context(1142            state_hash=sh0,1143            base_contexts=[],1144            inputs=[self._create_address('a')],1145            outputs=[self._create_address('a')])1146        self.assertEqual(self.context_manager.get(1147            ctx_1, [self._create_address('a')]),1148            [(self._create_address('a'), None)],1149            "ctx_1 has no value for 'a'")1150        self.context_manager.set(ctx_1, [{self._create_address('a'): b'1'}])1151        ctx_2a = self.context_manager.create_context(1152            state_hash=sh0,1153            base_contexts=[ctx_1],1154            inputs=[self._create_address('a')],1155            outputs=[self._create_address('b')])1156        self.assertEqual(self.context_manager.get(1157            ctx_2a, [self._create_address('a')]),1158            [(self._create_address('a'), b'1')],1159            "ctx_2a has the value b'1' for 'a'")1160        self.context_manager.set(ctx_2a, [{self._create_address('b'): b'2'}])1161        ctx_2b = self.context_manager.create_context(1162            state_hash=sh0,1163            base_contexts=[ctx_1],1164            inputs=[self._create_address('a')],1165            outputs=[self._create_address('c')])1166        self.assertEqual(self.context_manager.get(1167            ctx_2b, [self._create_address('a')]),1168            [(self._create_address('a'), b'1')],1169            "ctx_2b has the value b'1' for 'a'")1170        self.context_manager.set(ctx_2b, [{self._create_address('c'): b'2'}])1171        ctx_3 = self.context_manager.create_context(1172            state_hash=sh0,1173            base_contexts=[ctx_2b, ctx_2a],1174            inputs=[self._create_address('b')],1175            outputs=[self._create_address('b'), self._create_address('c')])1176        self.assertEqual(self.context_manager.get(1177            ctx_3, [self._create_address('b')]),1178            [(self._create_address('b'), b'2')],1179            "ctx_3 has the value b'2' for 'b'")1180        self.context_manager.delete(ctx_3, [self._create_address('b'),1181                                            self._create_address('c')])1182        ctx_4 = self.context_manager.create_context(1183            state_hash=sh0,1184            base_contexts=[ctx_3],1185            inputs=[1186                self._create_address('a'),1187                self._create_address('b'),1188                self._create_address('c')1189            ],1190            outputs=[self._create_address('c')])1191        self.assertEqual(self.context_manager.get(1192            ctx_4, [self._create_address('a'),1193                    self._create_address('b'),1194                    self._create_address('c')]),1195            [(self._create_address('a'), b'1'),1196             (self._create_address('b'), None),1197             (self._create_address('c'), None)],1198            "ctx_4 has the correct values in state for 'a','b', 'c'")1199        self.context_manager.set(ctx_4, [{self._create_address('c'): b'4'}])1200        squash = self.context_manager.get_squash_handler()1201        sh1 = squash(1202            state_root=sh0,1203            context_ids=[ctx_4],1204            persist=True,1205            clean_up=True)1206        tree = MerkleDatabase(self.database_results)1207        sh1_assertion = tree.update({1208            self._create_address('a'): b'1',1209            self._create_address('c'): b'4'1210        })1211        self.assertEqual(sh1, sh1_assertion,1212                         "The context manager must "1213                         "calculate the correct state hash")1214    def test_complex_read_write_delete(self):1215        """Tests complex reads, writes, and deletes from contexts.1216                  i:a1217                  o:b                           i:a1218               +->ctx_1a+                       o:d1219               |  s:b   |                    +->ctx_3a+1220               |        |                    |  d:d   |1221               |  i:a   |             i:a    |        |    i:""1222               |  o:c   |             o:""   |        |    o:""1223        sh0+----->ctx_1b<--->sh1+---->ctx_2+-+        <--->ctx_4+->sh21224               |  s:c   |             d:c    |  i:c   |1225               |  i:a   |             s:e    |  o:b   |1226               |  o:d   |                    +->ctx_3b+1227               +->ctx_1c+                       d:b1228                  s:d1229        Notes:1230            1. Aside from the initial state hash, there are two squashed1231               state hashes. There are sets in ctx_1* and then after a squash,1232               deletes of the same addresses.1233            2. Assertions are made for ctx_3b that 'c' is not in state, for1234               ctx_4 that 'b', 'c', and 'd' are not in state.1235        """1236        sh0 = self.first_state_hash1237        squash = self.context_manager.get_squash_handler()1238        ctx_1a = self.context_manager.create_context(1239            state_hash=sh0,1240            base_contexts=[],1241            inputs=[self._create_address('a')],1242            outputs=[self._create_address('b')])1243        ctx_1b = self.context_manager.create_context(1244            state_hash=sh0,1245            base_contexts=[],1246            inputs=[self._create_address('a')],1247            outputs=[self._create_address('c')])1248        ctx_1c = self.context_manager.create_context(1249            state_hash=sh0,1250            base_contexts=[],1251            inputs=[self._create_address('a')],1252            outputs=[self._create_address('d')])1253        self.context_manager.set(ctx_1a, [{self._create_address('b'): b'1'}])1254        self.context_manager.set(ctx_1b, [{self._create_address('c'): b'2'}])1255        self.context_manager.set(ctx_1c, [{self._create_address('d'): b'3'}])1256        sh1 = squash(1257            state_root=sh0,1258            context_ids=[ctx_1c, ctx_1b, ctx_1a],1259            persist=True,1260            clean_up=True)1261        ctx_2 = self.context_manager.create_context(1262            state_hash=sh1,1263            base_contexts=[],1264            inputs=[self._create_address('a')],1265            outputs=[""])1266        self.context_manager.delete(ctx_2, [self._create_address('c')])1267        self.context_manager.set(ctx_2, [{self._create_address('e'): b'2'}])1268        ctx_3a = self.context_manager.create_context(1269            state_hash=sh1,1270            base_contexts=[ctx_2],1271            inputs=[self._create_address('a')],1272            outputs=[self._create_address('d')])1273        self.context_manager.delete(ctx_3a, [self._create_address('d')])1274        ctx_3b = self.context_manager.create_context(1275            state_hash=sh1,1276            base_contexts=[ctx_2],1277            inputs=[self._create_address('c')],1278            outputs=[self._create_address('b')])1279        self.assertEqual(1280            self.context_manager.get(ctx_3b, [self._create_address('c')]),1281            [(self._create_address('c'), None)],1282            "Address 'c' has already been deleted from state.")1283        self.context_manager.delete(ctx_3b, [self._create_address('b')])1284        ctx_4 = self.context_manager.create_context(1285            state_hash=sh1,1286            base_contexts=[ctx_3b, ctx_3a],1287            inputs=[""],1288            outputs=[""])1289        self.assertEqual(1290            self.context_manager.get(ctx_4, [self._create_address('b'),1291                                             self._create_address('c'),1292                                             self._create_address('d')]),1293            [(self._create_address('b'), None),1294             (self._create_address('c'), None),1295             (self._create_address('d'), None)],1296            "Addresses 'b', 'c', and 'd' have been deleted from state.")1297        sh2 = squash(1298            state_root=sh1,1299            context_ids=[ctx_4],1300            persist=True,1301            clean_up=True)1302        tree = MerkleDatabase(self.database_results)1303        sh1_assertion = tree.update(1304            {1305                self._create_address('b'): b'1',1306                self._create_address('c'): b'2',1307                self._create_address('d'): b'3'1308            },1309            virtual=False)1310        self.assertEqual(sh1, sh1_assertion,1311                         "The middle state hash must be correct.")1312        tree.set_merkle_root(sh1)1313        sh2_assertion = tree.update(1314            {1315                self._create_address('e'): b'2'1316            },1317            delete_items=[1318                self._create_address('b'),1319                self._create_address('c'),1320                self._create_address('d')1321            ],1322            virtual=False)1323        self.assertEqual(sh2, sh2_assertion,...

test_schedulers_with_yaml.py

Source:test_schedulers_with_yaml.py

1# Copyright 2017 Intel Corporation2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14# ----------------------------------------------------------------------------15import unittest16import logging17import os18import shutil19import tempfile20from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase21from sawtooth_validator.state.merkle import MerkleDatabase22from sawtooth_validator.execution.context_manager import ContextManager23from sawtooth_validator.execution.scheduler_parallel import ParallelScheduler24from sawtooth_validator.execution.scheduler_serial import SerialScheduler25from test_scheduler.yaml_scheduler_tester import SchedulerTester26LOGGER = logging.getLogger(__name__)27class TestSchedulersWithYaml(unittest.TestCase):28    def __init__(self, test_name):29        super().__init__(test_name)30        self._temp_dir = None31    def setUp(self):32        self._temp_dir = tempfile.mkdtemp()33        database = NativeLmdbDatabase(34            os.path.join(self._temp_dir, 'test_state_view.lmdb'),35            indexes=MerkleDatabase.create_index_configuration(),36            _size=10 * 1024 * 1024)37        self._context_manager = ContextManager(database)38    def tearDown(self):39        self._context_manager.stop()40        shutil.rmtree(self._temp_dir)41    def _setup_serial_scheduler(self):42        context_manager = self._context_manager43        squash_handler = context_manager.get_squash_handler()44        first_state_root = context_manager.get_first_root()45        scheduler = SerialScheduler(squash_handler,46                                    first_state_root,47                                    always_persist=False)48        return context_manager, scheduler49    def _setup_parallel_scheduler(self):50        context_manager = self._context_manager51        squash_handler = context_manager.get_squash_handler()52        first_state_root = context_manager.get_first_root()53        scheduler = ParallelScheduler(squash_handler,54                                      first_state_root,55                                      always_persist=False)56        return context_manager, scheduler57    def _get_filenames(self):58        base_dir = os.path.abspath(os.path.dirname(__file__))59        data_dir = os.path.join(base_dir, 'data')60        filepaths = []61        for root, _, filenames in os.walk(data_dir):62            filepaths.extend(63                map(64                    lambda f, r=root: os.path.join(r, f),65                    filter(66                        lambda f: f.endswith('.yml') or f.endswith('.yaml'),67                        filenames)))68        return filepaths69    def test_all_yaml_files(self):70        """Tests the schedulers against each of the yaml files in the71        test_scheduler/data directory"""72        for file_path in self._get_filenames():73            try:74                context_manager, scheduler = self._setup_parallel_scheduler()75                self._single_block_files_individually(76                    scheduler=scheduler,77                    context_manager=context_manager,78                    name=file_path)79                context_manager, scheduler = self._setup_parallel_scheduler()80                self._single_block_files_individually(81                    scheduler=scheduler,82                    context_manager=context_manager,83                    name=file_path,84                    lifo=True)85                context_manager, scheduler = self._setup_parallel_scheduler()86                self._single_block_files_individually_alt(87                    scheduler=scheduler,88                    context_manager=context_manager,89                    name=file_path)90                context_manager, scheduler = self._setup_parallel_scheduler()91                self._single_block_files_individually_alt(92                    scheduler=scheduler,93                    context_manager=context_manager,94                    name=file_path,95                    lifo=True)96                context_manager, scheduler = self._setup_serial_scheduler()97                self._single_block_files_individually(98                    scheduler=scheduler,99                    context_manager=context_manager,100                    name=file_path)101                context_manager, scheduler = self._setup_serial_scheduler()102                self._single_block_files_individually_alt(103                    scheduler=scheduler,104                    context_manager=context_manager,105                    name=file_path)106            except AssertionError:107                LOGGER.warning("Failure on file %s", file_path)108                raise109    def _single_block_files_individually_alt(self,110                                             scheduler,111                                             context_manager,112                                             name,113                                             lifo=False):114        """Tests scheduler(s) with yaml files that represent a single115        block.116        Notes:117            Tests that the serial scheduler has the correct batch validity118            and state hash, and that 1 state hash is produced.119        """120        file_name = name121        tester = SchedulerTester(file_name)122        defined_batch_results_dict = tester.batch_results123        batch_results, txns_to_assert_state = tester.run_scheduler_alternating(124            scheduler=scheduler,125            context_manager=context_manager,126            txns_executed_fifo=not lifo)127        self._assertions(tester, defined_batch_results_dict,128                         batch_results,129                         txns_to_assert_state, name)130    def _single_block_files_individually(self,131                                         scheduler,132                                         context_manager,133                                         name,134                                         lifo=False):135        """Tests scheduler(s) with yaml files that represent a single136        block.137        Notes:138            Tests that the serial scheduler has the correct batch validity139            and state hash, and that 1 state hash is produced.140        """141        file_name = name142        tester = SchedulerTester(file_name)143        defined_batch_results_dict = tester.batch_results144        batch_results, txns_to_assert_state = tester.run_scheduler(145            scheduler=scheduler,146            context_manager=context_manager,147            txns_executed_fifo=not lifo)148        self._assertions(tester, defined_batch_results_dict,149                         batch_results,150                         txns_to_assert_state, name)151    def _assertions(self, tester, defined_batch_results_dict,152                    batch_results, txns_to_assert_state, name):153        self.assert_batch_validity(154            defined_batch_results_dict,155            batch_results)156        for t_id in txns_to_assert_state:157            state_and_txn_context = txns_to_assert_state.get(t_id)158            if state_and_txn_context is not None:159                txn_context, state_found, state_assert = state_and_txn_context160                self.assertEqual(state_found, state_assert,161                                 "Transaction {} in batch {} has the wrong "162                                 "state in the context".format(163                                     txn_context.txn_num,164                                     txn_context.batch_num))165        self.assert_one_state_hash(batch_results=batch_results)166        sched_state_roots = self._get_state_roots(167            batch_results=batch_results)168        calc_state_hash = tester.compute_state_hashes_wo_scheduler(169            self._temp_dir)170        self.assertEqual(171            sched_state_roots,172            calc_state_hash,173            "The state hashes calculated by the scheduler for yaml file {}"174            " must be the same as calculated by the tester".format(name))175    def _get_state_roots(self, batch_results):176        return [r.state_hash for _, r in batch_results177                if r.state_hash is not None]178    def assert_batch_validity(self, yaml_results_dict, batch_results):179        """Checks that all of the BatchExecutionResults calculated are the180        same as defined in the yaml file and returned by181        SchedulerTester.create_batches().182        Args:183            yaml_results_dict (dict): batch signature: BatchExecutionResult,184                                      Calculated from the yaml.185            batch_results (list): list of tuples186                                  (batch signature, BatchExecutionResult)187                                  Calculated by the scheduler.188        Raises:189            AssertionError190        """191        for i, b_e in enumerate(batch_results):192            signature, result = b_e193            defined_result = yaml_results_dict[signature]194            self.assertIsNotNone(result.is_valid, "Batch #{} has None as it's "195                                                  "validity".format(i))196            self.assertEqual(197                result.is_valid,198                defined_result.is_valid,199                "Batch #{} was defined in the yaml to be {},"200                "but the scheduler determined "201                "it was {}".format(202                    i + 1,203                    'valid' if defined_result.is_valid else 'invalid',204                    'valid' if result.is_valid else 'invalid'))205    def assert_one_state_hash(self, batch_results):206        """Should be used when only one state hash is expected from the207        scheduler.208        Args:209            batch_results (list): list of tuples210                                  (batch signature, BatchExecutionResult)211        Raises:212            AssertionError213        """214        state_roots = self._get_state_roots(batch_results=batch_results)215        self.assertEqual(len(state_roots), 1,216                         "The scheduler calculated more than one state "...

batch_thompson.py

Source:batch_thompson.py

1# Copyright (c) 2016, the GPyOpt Authors2# Licensed under the BSD 3-clause license (see LICENSE.txt)3from .base import SamplingBasedBatchEvaluator4from ...optimization.anchor_points_generator import ThompsonSamplingAnchorPointsGenerator5from ...optimization.optimizer import OptLbfgs, apply_optimizer, choose_optimizer6import numpy as np7class ThompsonBatch(SamplingBasedBatchEvaluator):8    """9    Class for a Thompson batch method. Elements are selected iteratively using the current acquistion function but exploring the models10    by using Thompson sampling11    :param acquisition: acquisition function to be used to compute the batch.12    :param batch size: the number of elements in the batch.13    """14    def __init__(self, acquisition, batch_size):15        super(ThompsonBatch, self).__init__(acquisition, batch_size)16        self.model              = self.acquisition.model17        self.optimizer_name     = 'lbfgs'18        self.f                  = self.acquisition.acquisition_function19        self.f_df               = self.acquisition.acquisition_function_withGradients20        self.space              = self.acquisition.space21    def initialize_batch(self, duplicate_manager=None, context_manager=None):22        return None23    def get_anchor_points(self, duplicate_manager=None, context_manager=None):24        design_type, unique = "random", False25        if duplicate_manager:26            unique = True27        anchor_points_generator = ThompsonSamplingAnchorPointsGenerator(self.space, design_type, model=self.model)28        return anchor_points_generator.get(num_anchor=self.num_anchor, duplicate_manager=duplicate_manager, unique=unique, context_manager = self.context_manager)29    def optimize_anchor_point(self, a, duplicate_manager=None, context_manager=None):30        ### --- Update the bounds of the default optimizer according to the context_manager31        if context_manager:32            bounds = self.context_manager.noncontext_bounds33        else:34            bounds = self.space.get_bounds()35        self.local_optimizer = choose_optimizer(self.optimizer_name, bounds)36        ### --- Run the local optimizer37        x, _ = apply_optimizer(self.local_optimizer, a, f=self.f, df=None, f_df=self.f_df, duplicate_manager=duplicate_manager, context_manager = self.context_manager, space=self.space)38        return self.space.round_optimum(x)39    def compute_batch_without_duplicate_logic(self, context_manager=None):40        anchor_points = self.get_anchor_points(context_manager=context_manager)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.