How to use context_manager method in pytest-mock

Best Python code snippet using pytest-mock

tests.py

Source:tests.py Github

copy

Full Screen

1# Copyright 2017 Intel Corporation2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14# ------------------------------------------------------------------------------15# pylint: disable=too-many-lines,broad-except16import unittest17from collections import namedtuple18import hashlib19import os20import shutil21import tempfile22import time23from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase24from sawtooth_validator.execution import context_manager25from sawtooth_validator.state.merkle import MerkleDatabase26from sawtooth_validator.protobuf.events_pb2 import Event27TestAddresses = namedtuple('TestAddresses',28 ['inputs', 'outputs', 'reads', 'writes'])29class TestContextManager(unittest.TestCase):30 def __init__(self, test_name):31 super().__init__(test_name)32 self._temp_dir = None33 def setUp(self):34 self._temp_dir = tempfile.mkdtemp()35 self.database_of_record = NativeLmdbDatabase(36 os.path.join(self._temp_dir, 'db_of_record.lmdb'),37 indexes=MerkleDatabase.create_index_configuration(),38 _size=10 * 1024 * 1024)39 self.context_manager = context_manager.ContextManager(40 self.database_of_record)41 self.first_state_hash = self.context_manager.get_first_root()42 # used for replicating state hash through direct merkle tree updates43 self.database_results = NativeLmdbDatabase(44 os.path.join(self._temp_dir, 'db_results.lmdb'),45 indexes=MerkleDatabase.create_index_configuration(),46 _size=10 * 1024 * 1024)47 def tearDown(self):48 self.context_manager.stop()49 shutil.rmtree(self._temp_dir)50 def _create_address(self, value=None):51 """52 Args:53 value: (str)54 Returns: (str) sha512 of value or random55 """56 if value is None:57 value = time.time().hex()58 return hashlib.sha512(value.encode()).hexdigest()[:70]59 def _setup_context(self):60 # 1) Create transaction data61 first_transaction = {'inputs': [self._create_address(a) for a in62 ['aaaa', 'bbbb', 'cccc']],63 'outputs': [self._create_address(a) for a in64 ['llaa', 'aall', 'nnnn']]}65 second_transaction = {66 'inputs': [self._create_address(a) for a in67 ['aaaa', 'dddd']],68 'outputs': [self._create_address(a) for a in69 ['zzzz', 'yyyy', 'tttt', 'qqqq']]70 }71 third_transaction = {72 'inputs': [self._create_address(a) for a in73 ['eeee', 'dddd', 'ffff']],74 'outputs': [self._create_address(a) for a in75 ['oooo', 'oozz', 'zzoo', 'ppoo', 'aeio']]76 }77 # 2) Create contexts based on that data78 context_id_1 = self.context_manager.create_context(79 state_hash=self.first_state_hash,80 base_contexts=[],81 inputs=first_transaction['inputs'],82 outputs=first_transaction['outputs'])83 context_id_2 = self.context_manager.create_context(84 state_hash=self.first_state_hash,85 base_contexts=[],86 inputs=second_transaction['inputs'],87 outputs=second_transaction['outputs'])88 context_id_3 = self.context_manager.create_context(89 state_hash=self.first_state_hash,90 base_contexts=[],91 inputs=third_transaction['inputs'],92 outputs=third_transaction['outputs'])93 # 3) Set addresses with values94 self.context_manager.set(context_id_1, [{self._create_address(a): v}95 for a, v in [('llaa', b'1'),96 ('aall', b'2'),97 ('nnnn', b'3')]])98 self.context_manager.set(context_id_2, [{self._create_address(a): v}99 for a, v in [('zzzz', b'9'),100 ('yyyy', b'11'),101 ('tttt', b'12'),102 ('qqqq', b'13')]])103 self.context_manager.set(context_id_3, [{self._create_address(a): v}104 for a, v in [('oooo', b'25'),105 ('oozz', b'26'),106 ('zzoo', b'27'),107 ('ppoo', b'28'),108 ('aeio', b'29')]])109 # 4)110 context_id = self.context_manager.create_context(111 state_hash=self.first_state_hash,112 base_contexts=[context_id_1, context_id_2, context_id_3],113 inputs=[114 self._create_address(a)115 for a in ['llaa', 'yyyy', 'tttt', 'zzoo']116 ],117 outputs=[118 self._create_address(a)119 for a in ['llaa', 'yyyy', 'tttt', 'zzoo', 'aeio']120 ])121 return context_id122 def _create_txn_inputs_outputs(self, start=None):123 """Create unique addresses that make up the inputs, outputs,124 reads, and writes that are involved in a context.125 Venn Diagram of relationship of disjoint sets that make up the126 inputs, outputs, reads, and writes.127 Knowledge of which disjoint set an address is a part of128 may give knowledge about a test failure in the context129 manager.130 Inputs Outputs131 +----------+--------------------------+-----------+132 | | | |133 | i___ |Reads io__ Writes _o__ |134 | | | |135 | +-----------+-----------+---------------+ |136 | | | | | | | |137 | | | | | | | |138 | | | | | | | |139 | | | | | | | |140 | |i_r_ | ior_| iorw | io_w | _o_w | |141 | | | | | | | |142 | | | | | | | |143 | | | | | | | |144 | +-----------+-----------+---------------+ |145 | | | |146 | | | |147 +----------+--------------------------+-----------+148 Args:149 start (int): An integer to start the sequence of integers being150 hashed to addresses.151 Returns (namedtuple): An object that holds inputs, outputs, reads,152 and writes.153 """154 if start is None:155 start = 0156 iorw = [self._create_address(str(i)) for i in range(start, start + 10)]157 i_r_ = [158 self._create_address(str(i))159 for i in range(start + 10, start + 20)160 ]161 ior_ = [162 self._create_address(str(i))163 for i in range(start + 20, start + 30)164 ]165 io__ = [166 self._create_address(str(i))167 for i in range(start + 30, start + 40)168 ]169 io_w = [170 self._create_address(str(i))171 for i in range(start + 40, start + 50)172 ]173 _o_w = [174 self._create_address(str(i))175 for i in range(start + 50, start + 60)176 ]177 _o__ = [178 self._create_address(str(i))179 for i in range(start + 60, start + 70)180 ]181 i___ = [182 self._create_address(str(i))183 for i in range(start + 70, start + 80)184 ]185 addresses = TestAddresses(186 inputs=iorw + ior_ + io__ + io_w + i___,187 outputs=ior_ + io__ + io_w + _o__ + _o_w,188 reads=i_r_ + ior_,189 writes=io_w + _o_w)190 return addresses191 def test_execution_results(self):192 """Tests that get_execution_results returns the correct values."""193 addr1 = self._create_address()194 addr2 = self._create_address()195 context_id = self.context_manager.create_context(196 state_hash=self.context_manager.get_first_root(),197 base_contexts=[],198 inputs=[addr1, addr2],199 outputs=[addr1, addr2])200 sets = {addr1: b'1'}201 events = [202 Event(203 event_type=teststr,204 attributes=[Event.Attribute(key=teststr, value=teststr)],205 data=teststr.encode()) for teststr in ("test1", "test2")206 ]207 deletes = {addr2: None}208 data = [(teststr.encode()) for teststr in ("test1", "test2")]209 self.context_manager.set(context_id, [sets])210 for event in events:211 self.context_manager.add_execution_event(context_id, event)212 self.context_manager.delete(context_id, deletes)213 for datum in data:214 self.context_manager.add_execution_data(215 context_id, datum)216 results = self.context_manager.get_execution_results(context_id)217 self.assertEqual(sets, results[0])218 self.assertEqual(deletes, results[1])219 self.assertEqual(events, results[2])220 self.assertEqual(data, results[3])221 def test_address_enforcement(self):222 """Tests that the ContextManager enforces address characteristics.223 Notes:224 1. Call get and set on the ContextManager with an address that is225 under a namespace, but is an invalid address, and test that226 the methods raise an AuthorizationException.227 """228 # 1)229 invalid_address1 = 'a' * 69 + 'n'230 invalid_address2 = 'b' * 69 + 'y'231 context_id1 = self.context_manager.create_context(232 state_hash=self.context_manager.get_first_root(),233 base_contexts=[],234 inputs=['aaaaaaaa', 'bbbbbbbb'],235 outputs=['aaaaaaaa', 'bbbbbbbb'])236 with self.assertRaises(context_manager.AuthorizationException):237 self.context_manager.get(238 context_id=context_id1,239 address_list=[invalid_address1, invalid_address2])240 with self.assertRaises(context_manager.AuthorizationException):241 self.context_manager.set(242 context_id=context_id1,243 address_value_list=[{invalid_address1: b'1'},244 {invalid_address2: b'2'}])245 def test_get_set_wrong_namespace(self):246 """Tests that getting and setting from outside the namespace will247 raise a AuthorizationException.248 Notes:249 1. Assert that sets on a context with addresses that aren't250 under an output namespace raise an AuthorizationException.251 2. Assert that gets on a context with addresses that aren't under252 an input namespace raise an AuthorizationException.253 """254 wrong_namespace1 = self._create_address('a')[-10:]255 wrong_namespace2 = '00000000'256 ctx_1 = self.context_manager.create_context(257 state_hash=self.context_manager.get_first_root(),258 base_contexts=[],259 inputs=[wrong_namespace1, wrong_namespace2],260 outputs=[wrong_namespace1, wrong_namespace2])261 # 1262 with self.assertRaises(context_manager.AuthorizationException):263 self.context_manager.set(264 context_id=ctx_1,265 address_value_list=[{self._create_address('a'): b'1'}])266 with self.assertRaises(context_manager.AuthorizationException):267 self.context_manager.set(268 context_id=ctx_1,269 address_value_list=[{self._create_address('c'): b'5'}])270 # 2271 with self.assertRaises(context_manager.AuthorizationException):272 self.context_manager.get(273 context_id=ctx_1,274 address_list=[self._create_address('a')])275 with self.assertRaises(context_manager.AuthorizationException):276 self.context_manager.get(277 context_id=ctx_1,278 address_list=[self._create_address('c')])279 def test_exception_on_invalid_input(self):280 """Tests that invalid inputs raise an exception. Tested with invalid281 characters, odd number of characters, and too long namespace;282 Notes:283 1) Assert that inputs with a namespace with an odd number of284 characters raise a CreateContextException.285 2) Assert that inputs with a 71 character namespace raise a286 CreateContextException.287 3) Assert that inputs with a namespace with several invalid288 characters raise a CreateContextException.289 """290 invalid_input_output1 = '0db7e8zc' # invalid character291 invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars292 invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars293 valid_input_output1 = 'd8f533bbb74443222daad4'294 valid_input_output2 = '77465847465784757848ddddddf'295 state_hash = self.context_manager.get_first_root()296 # 1297 with self.assertRaises(context_manager.CreateContextException):298 self.context_manager.create_context(299 state_hash=state_hash,300 base_contexts=[],301 inputs=[invalid_input_output1, valid_input_output1],302 outputs=[valid_input_output2])303 # 2304 with self.assertRaises(context_manager.CreateContextException):305 self.context_manager.create_context(306 state_hash=state_hash,307 base_contexts=[],308 inputs=[valid_input_output1, invalid_input_output2],309 outputs=[valid_input_output2])310 # 3311 with self.assertRaises(context_manager.CreateContextException):312 self.context_manager.create_context(313 state_hash=state_hash,314 base_contexts=[],315 inputs=[invalid_input_output3, valid_input_output2],316 outputs=[valid_input_output2, valid_input_output1])317 def test_exception_on_invalid_output(self):318 """Tests that invalid outputs raise an exception. Tested with invalid319 characters, odd number of characters, and too long namespace;320 Notes:321 1) Assert that outputs with a namespace with an odd number of322 characters raise a CreateContextException.323 2) Assert that outputs with a 71 character namespace raise a324 CreateContextException.325 3) Assert that outputs with a namespace with several invalid326 characters raise a CreateContextException.327 """328 invalid_input_output1 = '0db7e87' # Odd number of characters329 invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars330 invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars331 valid_input_output1 = 'd8f533bbb74443222daad4'332 valid_input_output2 = '77465847465784757848ddddddff'333 state_hash = self.context_manager.get_first_root()334 # 1335 with self.assertRaises(context_manager.CreateContextException):336 self.context_manager.create_context(337 state_hash=state_hash,338 base_contexts=[],339 inputs=[valid_input_output2, valid_input_output1],340 outputs=[invalid_input_output1])341 # 2342 with self.assertRaises(context_manager.CreateContextException):343 self.context_manager.create_context(344 state_hash=state_hash,345 base_contexts=[],346 inputs=[valid_input_output1, valid_input_output2],347 outputs=[invalid_input_output2])348 # 3349 with self.assertRaises(context_manager.CreateContextException):350 self.context_manager.create_context(351 state_hash=state_hash,352 base_contexts=[],353 inputs=[valid_input_output1, valid_input_output2],354 outputs=[valid_input_output2, invalid_input_output3])355 def test_namespace_gets(self):356 """Tests that gets for an address under a namespace will return the357 correct value.358 Notes:359 1) Create ctx_1 and set 'b' to b'8'.360 2) squash the previous context creating state_hash_1.361 3) Create 2 contexts off of this state hash and assert362 that gets on these contexts retrieve the correct363 value for an address that is not fully specified in the inputs.364 4) Set values to addresses in these contexts.365 5) Create 1 context off of these prior 2 contexts and assert that366 gets from this context retrieve the correct values for367 addresses that are not fully specified in the inputs. 2 of the368 values are found in the chain of contexts, and 1 is not found369 and so must be retrieved from the merkle tree.370 """371 # 1372 ctx_1 = self.context_manager.create_context(373 state_hash=self.context_manager.get_first_root(),374 base_contexts=[],375 inputs=[self._create_address('a')],376 outputs=[self._create_address('b')])377 self.context_manager.set(378 context_id=ctx_1,379 address_value_list=[{self._create_address('b'): b'8'}])380 # 2381 squash = self.context_manager.get_squash_handler()382 state_hash_1 = squash(383 state_root=self.context_manager.get_first_root(),384 context_ids=[ctx_1],385 persist=True,386 clean_up=True)387 # 3388 ctx_1a = self.context_manager.create_context(389 state_hash=state_hash_1,390 base_contexts=[],391 inputs=[self._create_address('a')[:10]],392 outputs=[self._create_address('c')])393 self.assertEqual(394 self.context_manager.get(395 context_id=ctx_1a,396 address_list=[self._create_address('a')]),397 [(self._create_address('a'), None)])398 ctx_1b = self.context_manager.create_context(399 state_hash=state_hash_1,400 base_contexts=[],401 inputs=[self._create_address('b')[:6]],402 outputs=[self._create_address('z')])403 self.assertEqual(404 self.context_manager.get(405 context_id=ctx_1b,406 address_list=[self._create_address('b')]),407 [(self._create_address('b'), b'8')])408 # 4409 self.context_manager.set(410 context_id=ctx_1b,411 address_value_list=[{self._create_address('z'): b'2'}])412 self.context_manager.set(413 context_id=ctx_1a,414 address_value_list=[{self._create_address('c'): b'1'}]415 )416 ctx_2 = self.context_manager.create_context(417 state_hash=state_hash_1,418 base_contexts=[ctx_1a, ctx_1b],419 inputs=[420 self._create_address('z')[:10],421 self._create_address('c')[:10],422 self._create_address('b')[:10]423 ],424 outputs=[self._create_address('w')])425 self.assertEqual(426 self.context_manager.get(427 context_id=ctx_2,428 address_list=[self._create_address('z'),429 self._create_address('c'),430 self._create_address('b')]),431 [(self._create_address('z'), b'2'),432 (self._create_address('c'), b'1'),433 (self._create_address('b'), b'8')])434 def test_create_context_with_prior_state(self):435 """Tests context creation with prior state from base contexts.436 Notes:437 Set up the context:438 Create 3 prior contexts each with 3-5 addresses to set to.439 Make set calls to those addresses.440 Create 1 new context based on those three prior contexts.441 this test method:442 Test:443 Make a get call on addresses that are from prior state,444 making assertions about the correct values.445 """446 context_id = self._setup_context()447 self.assertEqual(self.context_manager.get(448 context_id,449 [self._create_address(a) for a in450 ['llaa', 'yyyy', 'tttt', 'zzoo']]),451 [(self._create_address(a), v) for a, v in452 [('llaa', b'1'),453 ('yyyy', b'11'),454 ('tttt', b'12'),455 ('zzoo', b'27')]])456 def test_squash(self):457 """Tests that squashing a context based on state from other458 contexts will result in the same merkle hash as updating the459 merkle tree with the same data.460 Notes:461 Set up the context462 Test:463 1) Make set calls on several of the addresses.464 2) Squash the context to get a new state hash.465 3) Apply all of the aggregate sets from all466 of the contexts, to another database with a merkle tree.467 4) Assert that the state hashes are the same.468 """469 # 1)470 context_id = self._setup_context()471 self.context_manager.set(472 context_id,473 [{self._create_address(a): v} for a, v in474 [('yyyy', b'2'),475 ('tttt', b'4')]])476 # 2)477 squash = self.context_manager.get_squash_handler()478 resulting_state_hash = squash(self.first_state_hash, [context_id],479 persist=True, clean_up=True)480 # 3)481 final_state_to_update = {self._create_address(a): v for a, v in482 [('llaa', b'1'),483 ('aall', b'2'),484 ('nnnn', b'3'),485 ('zzzz', b'9'),486 ('yyyy', b'2'),487 ('tttt', b'4'),488 ('qqqq', b'13'),489 ('oooo', b'25'),490 ('oozz', b'26'),491 ('zzoo', b'27'),492 ('ppoo', b'28'),493 ('aeio', b'29')]}494 test_merkle_tree = MerkleDatabase(self.database_results)495 test_resulting_state_hash = test_merkle_tree.update(496 final_state_to_update, virtual=False)497 # 4)498 self.assertEqual(resulting_state_hash, test_resulting_state_hash)499 def test_squash_no_updates(self):500 """Tests that squashing a context that has no state updates will return501 the starting state root hash.502 Notes:503 Set up the context504 Test:505 1) Squash the context.506 2) Assert that the state hash is the same as the starting507 hash.508 """509 context_id = self.context_manager.create_context(510 state_hash=self.first_state_hash,511 base_contexts=[],512 inputs=[],513 outputs=[])514 # 1)515 squash = self.context_manager.get_squash_handler()516 resulting_state_hash = squash(self.first_state_hash, [context_id],517 persist=True, clean_up=True)518 # 2519 self.assertIsNotNone(resulting_state_hash)520 self.assertEqual(resulting_state_hash, self.first_state_hash)521 def test_squash_deletes_no_update(self):522 """Tests that squashing a context that has no state updates,523 due to sets that were subsequently deleted, will return524 the starting state root hash.525 Notes:526 Set up the context527 Test:528 1) Send Updates that reverse each other.529 2) Squash the context.530 3) Assert that the state hash is the same as the starting531 hash.532 """533 context_id = self.context_manager.create_context(534 state_hash=self.first_state_hash,535 base_contexts=[],536 inputs=[],537 outputs=[self._create_address(a) for a in538 ['yyyy', 'tttt']])539 # 1)540 self.context_manager.set(541 context_id,542 [{self._create_address(a): v} for a, v in543 [('yyyy', b'2'),544 ('tttt', b'4')]])545 self.context_manager.delete(546 context_id,547 [self._create_address(a) for a in548 ['yyyy', 'tttt']])549 # 2)550 squash = self.context_manager.get_squash_handler()551 resulting_state_hash = squash(self.first_state_hash, [context_id],552 persist=True, clean_up=True)553 # 3)554 self.assertIsNotNone(resulting_state_hash)555 self.assertEqual(resulting_state_hash, self.first_state_hash)556 def test_reads_from_context_w_several_writes(self):557 """Tests that those context values that have been written to the558 Merkle tree, or that have been set to a base_context, will have the559 correct value at the address for a given context.560 ->context_id_a1561 | |562 | |563 | |564 sh0-->context_id1-->sh1-->context_a----- -->context_id_b565 | |566 | |567 | |568 | |569 -->context_id_a2570 Notes:571 Test:572 1. From a Merkle Tree with only the root node in it, create a573 context and set several values, and then squash that context574 upon the first state hash.575 2. Create a context with no base context, based on576 the merkle root computed from the first squash.577 Assert that gets from this context will provide578 values that were set in the first context.579 3. Write to all of the available outputs580 4. Create a new context based on context_a, from #2,581 5. Assert that gets from this context equal the values set582 to Context A.583 6. Create a new context based on context_a and set values to584 this context.585 7. Create a new context based on the 2 contexts made in 4 and 6586 8. From this context assert that gets equal the correct values587 set in the prior contexts.588 """589 squash = self.context_manager.get_squash_handler()590 test_addresses = self._create_txn_inputs_outputs()591 # 1)592 context_id1 = self.context_manager.create_context(593 state_hash=self.first_state_hash,594 inputs=test_addresses.inputs,595 outputs=test_addresses.outputs,596 base_contexts=[])597 values1 = [bytes(i) for i in range(len(test_addresses.writes))]598 self.context_manager.set(599 context_id1,600 [{a: v} for a, v in zip(test_addresses.writes, values1)])601 sh1 = squash(602 state_root=self.first_state_hash,603 context_ids=[context_id1],604 persist=True,605 clean_up=True)606 # 2)607 context_a = self.context_manager.create_context(608 state_hash=sh1,609 inputs=test_addresses.writes, # read from every address written to610 outputs=test_addresses.outputs,611 base_contexts=[]612 )613 address_values = self.context_manager.get(614 context_a,615 list(test_addresses.writes)616 )617 self.assertEqual(618 address_values,619 list(zip(test_addresses.writes, values1))620 )621 # 3)622 values2 = [bytes(v.encode()) for v in test_addresses.outputs]623 self.context_manager.set(624 context_id=context_a,625 address_value_list=[dict(zip(test_addresses.outputs, values2))])626 # 4)627 context_id_a1 = self.context_manager.create_context(628 state_hash=sh1,629 inputs=test_addresses.outputs,630 outputs=test_addresses.outputs,631 base_contexts=[context_a]632 )633 # 5)634 c_ida1_address_values = self.context_manager.get(635 context_id=context_id_a1,636 address_list=list(test_addresses.outputs)637 )638 self.assertEqual(639 c_ida1_address_values,640 list(zip(test_addresses.outputs, values2))641 )642 # 6)643 test_addresses2 = self._create_txn_inputs_outputs(80)644 context_id_a2 = self.context_manager.create_context(645 state_hash=sh1,646 inputs=test_addresses2.inputs,647 outputs=test_addresses2.outputs,648 base_contexts=[context_a]649 )650 values3 = [bytes(v.encode()) for v in test_addresses2.writes]651 self.context_manager.set(652 context_id=context_id_a2,653 address_value_list=[{a: v} for654 a, v in zip(test_addresses2.writes, values3)],655 )656 # 7)657 context_id_b = self.context_manager.create_context(658 state_hash=sh1,659 inputs=test_addresses2.writes + test_addresses.outputs,660 outputs=[],661 base_contexts=[context_id_a1, context_id_a2]662 )663 # 8)664 self.assertEqual(665 self.context_manager.get(666 context_id_b,667 list(test_addresses2.writes + test_addresses.outputs)668 ),669 list(zip(670 test_addresses2.writes + test_addresses.outputs,671 values3 + values2))672 )673 def test_state_root_after_parallel_ctx(self):674 """Tests that the correct state root is calculated after basing one675 context off of multiple contexts.676 i=abcd677 o=aaaa678 +>context_1+679 | aaaa=1 |680 | |681 i=llll | i=bacd | i=bbbb,aaaa682 o=llll | o=bbbb | o=cccc,llll683 sh0--->ctx_0-->sh1>|-->context_2-+---->context_n---->sh2684 llll=5 | bbbb=2 | cccc=4685 | | llll=8686 | i=abcd |687 | o=cccc |688 +>context_3+689 cccc=3690 Notes:691 Test:692 1. Create a context, set a value in it and squash it into a new693 state hash.694 2. Create 3 contexts based off of the state root from #1.695 3. Set values at addresses to all three contexts.696 4. Base another context off of the contexts from #2.697 5. Set a value to an address in this context that has already698 been set to in the non-base context.699 6. Squash the contexts producing a state hash and assert700 that it equals a state hash obtained by manually updating701 the merkle tree.702 """703 sh0 = self.first_state_hash704 # 1)705 squash = self.context_manager.get_squash_handler()706 ctx_1 = self.context_manager.create_context(707 state_hash=sh0,708 base_contexts=[],709 inputs=[self._create_address('llll')],710 outputs=[self._create_address('llll')]711 )712 self.context_manager.set(713 context_id=ctx_1,714 address_value_list=[{self._create_address('llll'): b'5'}]715 )716 sh1 = squash(717 state_root=sh0,718 context_ids=[ctx_1],719 persist=True,720 clean_up=True)721 # 2)722 context_1 = self.context_manager.create_context(723 state_hash=sh1,724 base_contexts=[],725 inputs=[self._create_address('abcd')],726 outputs=[self._create_address('aaaa')])727 context_2 = self.context_manager.create_context(728 state_hash=sh1,729 base_contexts=[],730 inputs=[self._create_address('bacd')],731 outputs=[self._create_address('bbbb')])732 context_3 = self.context_manager.create_context(733 state_hash=sh1,734 base_contexts=[],735 inputs=[self._create_address('abcd')],736 outputs=[737 self._create_address('cccc'),738 self._create_address('dddd')739 ])740 # 3)741 self.context_manager.set(742 context_id=context_1,743 address_value_list=[{self._create_address('aaaa'): b'1'}]744 )745 self.context_manager.set(746 context_id=context_2,747 address_value_list=[{self._create_address('bbbb'): b'2'}]748 )749 self.context_manager.set(750 context_id=context_3,751 address_value_list=[{self._create_address('cccc'): b'3'}]752 )753 # 4)754 context_n = self.context_manager.create_context(755 state_hash=sh1,756 base_contexts=[context_1, context_2, context_3],757 inputs=[758 self._create_address('bbbb'),759 self._create_address('aaaa')760 ],761 outputs=[762 self._create_address('cccc'),763 self._create_address('llll')764 ])765 # 5)766 self.context_manager.set(767 context_id=context_n,768 address_value_list=[{769 self._create_address('cccc'): b'4',770 self._create_address('llll'): b'8'771 }])772 # 6)773 cm_state_root = squash(774 state_root=sh1,775 context_ids=[context_n],776 persist=False,777 clean_up=True)778 tree = MerkleDatabase(self.database_results)779 calc_state_root = tree.update({780 self._create_address('aaaa'): b'1',781 self._create_address('bbbb'): b'2',782 self._create_address('cccc'): b'4',783 self._create_address('llll'): b'8'784 })785 self.assertEqual(calc_state_root, cm_state_root)786 def test_complex_basecontext_squash(self):787 """Tests complex context basing and squashing.788 i=qq,dd dd=0789 o=dd,pp pp=1790 i=cc,aa +->context_3_2a_1+|791 o=dd,ll | |792 i=aa,ab +->context_2a| i=aa aa=0 |793 o=cc,ab | dd=10 | o=aa,ll ll=1 |794 sh0->context_1-->sh1| ll=11 +->context_3_2a_2+|->sh1795 cc=0 | i=cc,aa +->context_3_2b_1+|796 ab=1 | o=nn,mm | i=nn,ba mm=0 |797 +->context_2b| o=mm,ba ba=1 |798 nn=0 | |799 mm=1 +->context_3_2b_2+|800 i=nn,oo ab=0801 o=ab,oo oo=1802 Notes:803 Test:804 1. Create a context off of the first state hash, set805 addresses in it, and squash that context, getting a new806 merkle root.807 2. Create 2 contexts with the context in # 1 as the base, and808 for each of these contexts set addresses to values where the809 outputs for each are disjoint.810 3. For each of these 2 contexts create 2 more contexts each811 having one of the contexts in # 2 as the base context, and812 set addresses to values.813 4. Squash the 4 contexts from #3 and assert the state hash814 is equal to a manually computed state hash.815 """816 squash = self.context_manager.get_squash_handler()817 # 1)818 inputs_1 = [self._create_address('aa'),819 self._create_address('ab')]820 outputs_1 = [self._create_address('cc'),821 self._create_address('ab')]822 context_1 = self.context_manager.create_context(823 state_hash=self.first_state_hash,824 base_contexts=[],825 inputs=inputs_1,826 outputs=outputs_1)827 self.context_manager.set(828 context_id=context_1,829 address_value_list=[{a: v} for a, v in zip(830 outputs_1, [bytes(i) for i in range(len(outputs_1))])])831 sh1 = squash(832 state_root=self.first_state_hash,833 context_ids=[context_1],834 persist=True,835 clean_up=True)836 # 2)837 inputs_2a = [self._create_address('cc'),838 self._create_address('aa')]839 outputs_2a = [self._create_address('dd'),840 self._create_address('ll')]841 context_2a = self.context_manager.create_context(842 state_hash=self.first_state_hash,843 base_contexts=[],844 inputs=inputs_2a,845 outputs=outputs_2a)846 inputs_2b = [self._create_address('cc'),847 self._create_address('aa')]848 outputs_2b = [self._create_address('nn'),849 self._create_address('mm')]850 context_2b = self.context_manager.create_context(851 state_hash=sh1,852 base_contexts=[],853 inputs=inputs_2b,854 outputs=outputs_2b)855 self.context_manager.set(856 context_id=context_2a,857 address_value_list=[{a: bytes(v)}858 for a, v in zip(outputs_2a,859 range(10,860 10 + len(outputs_2a)))]861 )862 self.context_manager.set(863 context_id=context_2b,864 address_value_list=[{a: bytes(v)}865 for a, v in zip(outputs_2b,866 range(len(outputs_2b)))]867 )868 # 3)869 inputs_3_2a_1 = [870 self._create_address('qq'),871 self._create_address('dd')872 ]873 outputs_3_2a_1 = [874 self._create_address('dd'),875 self._create_address('pp')876 ]877 context_3_2a_1 = self.context_manager.create_context(878 state_hash=sh1,879 base_contexts=[context_2a],880 inputs=inputs_3_2a_1,881 outputs=outputs_3_2a_1)882 inputs_3_2a_2 = [self._create_address('aa')]883 outputs_3_2a_2 = [884 self._create_address('aa'),885 self._create_address('ll')886 ]887 context_3_2a_2 = self.context_manager.create_context(888 state_hash=sh1,889 base_contexts=[context_2a],890 inputs=inputs_3_2a_2,891 outputs=outputs_3_2a_2)892 inputs_3_2b_1 = [893 self._create_address('nn'),894 self._create_address('ab')895 ]896 outputs_3_2b_1 = [897 self._create_address('mm'),898 self._create_address('ba')899 ]900 context_3_2b_1 = self.context_manager.create_context(901 state_hash=sh1,902 base_contexts=[context_2b],903 inputs=inputs_3_2b_1,904 outputs=outputs_3_2b_1)905 inputs_3_2b_2 = [906 self._create_address('nn'),907 self._create_address('oo')908 ]909 outputs_3_2b_2 = [910 self._create_address('ab'),911 self._create_address('oo')912 ]913 context_3_2b_2 = self.context_manager.create_context(914 state_hash=sh1,915 base_contexts=[context_2b],916 inputs=inputs_3_2b_2,917 outputs=outputs_3_2b_2)918 self.context_manager.set(919 context_id=context_3_2a_1,920 address_value_list=[{a: bytes(v)}921 for a, v in zip(outputs_3_2a_1,922 range(len(outputs_3_2a_1)))])923 self.context_manager.set(924 context_id=context_3_2a_2,925 address_value_list=[{a: bytes(v)}926 for a, v in zip(outputs_3_2a_2,927 range(len(outputs_3_2a_2)))])928 self.context_manager.set(929 context_id=context_3_2b_1,930 address_value_list=[{a: bytes(v)}931 for a, v in zip(outputs_3_2b_1,932 range(len(outputs_3_2b_1)))])933 self.context_manager.set(934 context_id=context_3_2b_2,935 address_value_list=[{a: bytes(v)}936 for a, v in zip(outputs_3_2b_2,937 range(len(outputs_3_2b_2)))])938 # 4)939 sh2 = squash(940 state_root=sh1,941 context_ids=[context_3_2a_1, context_3_2a_2,942 context_3_2b_1, context_3_2b_2],943 persist=False,944 clean_up=True)945 tree = MerkleDatabase(self.database_results)946 state_hash_from_1 = tree.update(947 set_items=dict(zip(outputs_1,948 [bytes(i) for i in range(len(outputs_1))])),949 virtual=False)950 self.assertEqual(state_hash_from_1, sh1,951 "The manually calculated state hash from the first "952 "context and the one calculated by squashing that "953 "state hash should be the same")954 tree.set_merkle_root(state_hash_from_1)955 test_sh2 = tree.update(956 set_items={957 self._create_address('aa'): bytes(0),958 self._create_address('ab'): bytes(0),959 self._create_address('ba'): bytes(1),960 self._create_address('dd'): bytes(0),961 self._create_address('ll'): bytes(1),962 self._create_address('mm'): bytes(0),963 self._create_address('oo'): bytes(1),964 self._create_address('pp'): bytes(1),965 self._create_address('nn'): bytes(0),966 self._create_address('cc'): bytes(0)})967 self.assertEqual(sh2, test_sh2, "Manually calculated and context "968 "manager calculated merkle hashes "969 "are the same")970 def test_wildcarded_inputs_outputs(self):971 """Tests the context manager with wildcarded inputs and outputs.972 Notes:973 1. Create a context with a wildcarded input and output and974 another non-wildcarded input and output.975 2. Get an address under the wildcard and set to both the976 non-wildcarded address and an address under the977 wildcard.978 3. Squash the context and compare to a manually generated979 state hash.980 """981 # 1982 namespaces = [983 self._create_address('a')[:8],984 self._create_address('b')985 ]986 ctx_1 = self.context_manager.create_context(987 inputs=namespaces,988 outputs=namespaces,989 base_contexts=[],990 state_hash=self.first_state_hash)991 # 2992 self.context_manager.get(993 context_id=ctx_1,994 address_list=[self._create_address('a')])995 self.context_manager.set(996 context_id=ctx_1,997 address_value_list=[{998 self._create_address('a'): b'1',999 self._create_address('b'): b'2'1000 }])1001 # 31002 squash = self.context_manager.get_squash_handler()1003 tree = MerkleDatabase(self.database_results)1004 tree.set_merkle_root(self.first_state_hash)1005 calculated_state_root = tree.update(1006 set_items={self._create_address('a'): b'1',1007 self._create_address('b'): b'2'})1008 state_root = squash(1009 state_root=self.first_state_hash,1010 context_ids=[ctx_1],1011 persist=True,1012 clean_up=True)1013 self.assertEqual(state_root, calculated_state_root)1014 @unittest.skip("Necessary to catch scheduler bugs--Depth-first search")1015 def test_check_for_bad_combination(self):1016 """Tests that the context manager will raise1017 an exception if asked to combine contexts, either via base contexts1018 in create_context or via squash that shouldn't be1019 combined because they share addresses that can't be determined by the1020 scheduler to not have been parallel. This is a check on scheduler bugs.1021 Examples where the context manager should raise an exception on1022 duplicate addresses:1023 1. Success1024 i=a1025 o=b1026 +>ctx_1+1027 dup|->b=3 |1028 | |1029 sh0| ----->state hash or context1030 | i=q |1031 | o=b |1032 +>ctx_2+1033 dup-->b=21034 2.1035 i=b1036 o=d1037 +>ctx_1a_1+1038 | d=4 |1039 i=a | |1040 o=b | |1041 +>ctx_1a| |1042 | b=2 | i=b |1043 | | o=c |1044 sh0| +>ctx_1a_2|1045 | dup-> c=7 |------>state hash or context1046 | i=a +>ctx_1b_1|1047 | o=c | |1048 +>ctx_1b| |1049 dup-->c=5 | i=t |1050 | o=p |1051 +>ctx_1b_2+1052 p=81053 3.1054 i=b1055 o=d1056 +>ctx_1a_1+1057 | d=4 | i=d,c1058 i=a | | o=n1059 o=b | <>ctx_3a+1060 +>ctx_1a| | n=5 |1061 | b=2 | i=b | |1062 | | o=c | |1063 sh0| +>ctx_1a_2+ <----->state hash or context1064 | dup--> c=7 |1065 | i=a +>ctx_1b_1+ |1066 | o=c | | i=c |1067 +>ctx_1b| | o=q |1068 c=5 | i=c <>ctx_3b+1069 | o=c | q=51070 +>ctx_1b_2+1071 dup--> c=11072 """1073 # 1.1074 squash = self.context_manager.get_squash_handler()1075 sh0 = self.first_state_hash1076 inputs_1 = [self._create_address('a')]1077 outputs_1 = [self._create_address('b')]1078 ctx_1 = self.context_manager.create_context(1079 state_hash=sh0,1080 base_contexts=[],1081 inputs=inputs_1,1082 outputs=outputs_11083 )1084 self.context_manager.set(1085 context_id=ctx_1,1086 address_value_list=[{self._create_address('b'): b'3'}]1087 )1088 inputs_2 = [self._create_address('q')]1089 outputs_2 = [self._create_address('b')]1090 ctx_2 = self.context_manager.create_context(1091 state_hash=sh0,1092 base_contexts=[],1093 inputs=inputs_2,1094 outputs=outputs_2)1095 self.context_manager.set(1096 context_id=ctx_2,1097 address_value_list=[{1098 self._create_address('b'): b'2'1099 }])1100 try:1101 squash(1102 state_root=sh0,1103 context_ids=[ctx_1, ctx_2],1104 persist=True,1105 clean_up=True)1106 self.fail("squash of two contexts with a duplicate address")1107 except Exception:1108 pass1109 def test_simple_read_write_delete(self):1110 """Tests that after sets and deletes, subsequent contexts have the1111 correct value in the context for an address.1112 i:a1113 o:b1114 +-->ctx_2a+1115 | s:b:2 |1116 | |1117 i:a | | i:b i:a,b,c1118 o:a | | o:b,c o:c1119 sh0+->ctx_1| <--->ctx_3+---->ctx_4+------>sh11120 s:a:1| | d:b,c s:c:41121 | |1122 | i:a |1123 | o:c |1124 +-->ctx_2b+1125 s:c:21126 Notes:1127 1. From the diagram:1128 - ctx_1, with input 'a' and output 'a' will have 'a' set to1129 b'1'.1130 - ctx_2a will have 'b' set to b'2'1131 - ctx_2b will have 'c' set to b'2'1132 - ctx_3 will delete both 'b' and 'c'1133 - ctx_4 will set 'c' to b'4'1134 - ctx_4 will be squashed along with it's base contexts into1135 a new state hash.1136 2. Assertions1137 - Assert for every context that it has the correct state1138 before the set or delete.1139 """1140 sh0 = self.context_manager.get_first_root()1141 ctx_1 = self.context_manager.create_context(1142 state_hash=sh0,1143 base_contexts=[],1144 inputs=[self._create_address('a')],1145 outputs=[self._create_address('a')])1146 self.assertEqual(self.context_manager.get(1147 ctx_1, [self._create_address('a')]),1148 [(self._create_address('a'), None)],1149 "ctx_1 has no value for 'a'")1150 self.context_manager.set(ctx_1, [{self._create_address('a'): b'1'}])1151 ctx_2a = self.context_manager.create_context(1152 state_hash=sh0,1153 base_contexts=[ctx_1],1154 inputs=[self._create_address('a')],1155 outputs=[self._create_address('b')])1156 self.assertEqual(self.context_manager.get(1157 ctx_2a, [self._create_address('a')]),1158 [(self._create_address('a'), b'1')],1159 "ctx_2a has the value b'1' for 'a'")1160 self.context_manager.set(ctx_2a, [{self._create_address('b'): b'2'}])1161 ctx_2b = self.context_manager.create_context(1162 state_hash=sh0,1163 base_contexts=[ctx_1],1164 inputs=[self._create_address('a')],1165 outputs=[self._create_address('c')])1166 self.assertEqual(self.context_manager.get(1167 ctx_2b, [self._create_address('a')]),1168 [(self._create_address('a'), b'1')],1169 "ctx_2b has the value b'1' for 'a'")1170 self.context_manager.set(ctx_2b, [{self._create_address('c'): b'2'}])1171 ctx_3 = self.context_manager.create_context(1172 state_hash=sh0,1173 base_contexts=[ctx_2b, ctx_2a],1174 inputs=[self._create_address('b')],1175 outputs=[self._create_address('b'), self._create_address('c')])1176 self.assertEqual(self.context_manager.get(1177 ctx_3, [self._create_address('b')]),1178 [(self._create_address('b'), b'2')],1179 "ctx_3 has the value b'2' for 'b'")1180 self.context_manager.delete(ctx_3, [self._create_address('b'),1181 self._create_address('c')])1182 ctx_4 = self.context_manager.create_context(1183 state_hash=sh0,1184 base_contexts=[ctx_3],1185 inputs=[1186 self._create_address('a'),1187 self._create_address('b'),1188 self._create_address('c')1189 ],1190 outputs=[self._create_address('c')])1191 self.assertEqual(self.context_manager.get(1192 ctx_4, [self._create_address('a'),1193 self._create_address('b'),1194 self._create_address('c')]),1195 [(self._create_address('a'), b'1'),1196 (self._create_address('b'), None),1197 (self._create_address('c'), None)],1198 "ctx_4 has the correct values in state for 'a','b', 'c'")1199 self.context_manager.set(ctx_4, [{self._create_address('c'): b'4'}])1200 squash = self.context_manager.get_squash_handler()1201 sh1 = squash(1202 state_root=sh0,1203 context_ids=[ctx_4],1204 persist=True,1205 clean_up=True)1206 tree = MerkleDatabase(self.database_results)1207 sh1_assertion = tree.update({1208 self._create_address('a'): b'1',1209 self._create_address('c'): b'4'1210 })1211 self.assertEqual(sh1, sh1_assertion,1212 "The context manager must "1213 "calculate the correct state hash")1214 def test_complex_read_write_delete(self):1215 """Tests complex reads, writes, and deletes from contexts.1216 i:a1217 o:b i:a1218 +->ctx_1a+ o:d1219 | s:b | +->ctx_3a+1220 | | | d:d |1221 | i:a | i:a | | i:""1222 | o:c | o:"" | | o:""1223 sh0+----->ctx_1b<--->sh1+---->ctx_2+-+ <--->ctx_4+->sh21224 | s:c | d:c | i:c |1225 | i:a | s:e | o:b |1226 | o:d | +->ctx_3b+1227 +->ctx_1c+ d:b1228 s:d1229 Notes:1230 1. Aside from the initial state hash, there are two squashed1231 state hashes. There are sets in ctx_1* and then after a squash,1232 deletes of the same addresses.1233 2. Assertions are made for ctx_3b that 'c' is not in state, for1234 ctx_4 that 'b', 'c', and 'd' are not in state.1235 """1236 sh0 = self.first_state_hash1237 squash = self.context_manager.get_squash_handler()1238 ctx_1a = self.context_manager.create_context(1239 state_hash=sh0,1240 base_contexts=[],1241 inputs=[self._create_address('a')],1242 outputs=[self._create_address('b')])1243 ctx_1b = self.context_manager.create_context(1244 state_hash=sh0,1245 base_contexts=[],1246 inputs=[self._create_address('a')],1247 outputs=[self._create_address('c')])1248 ctx_1c = self.context_manager.create_context(1249 state_hash=sh0,1250 base_contexts=[],1251 inputs=[self._create_address('a')],1252 outputs=[self._create_address('d')])1253 self.context_manager.set(ctx_1a, [{self._create_address('b'): b'1'}])1254 self.context_manager.set(ctx_1b, [{self._create_address('c'): b'2'}])1255 self.context_manager.set(ctx_1c, [{self._create_address('d'): b'3'}])1256 sh1 = squash(1257 state_root=sh0,1258 context_ids=[ctx_1c, ctx_1b, ctx_1a],1259 persist=True,1260 clean_up=True)1261 ctx_2 = self.context_manager.create_context(1262 state_hash=sh1,1263 base_contexts=[],1264 inputs=[self._create_address('a')],1265 outputs=[""])1266 self.context_manager.delete(ctx_2, [self._create_address('c')])1267 self.context_manager.set(ctx_2, [{self._create_address('e'): b'2'}])1268 ctx_3a = self.context_manager.create_context(1269 state_hash=sh1,1270 base_contexts=[ctx_2],1271 inputs=[self._create_address('a')],1272 outputs=[self._create_address('d')])1273 self.context_manager.delete(ctx_3a, [self._create_address('d')])1274 ctx_3b = self.context_manager.create_context(1275 state_hash=sh1,1276 base_contexts=[ctx_2],1277 inputs=[self._create_address('c')],1278 outputs=[self._create_address('b')])1279 self.assertEqual(1280 self.context_manager.get(ctx_3b, [self._create_address('c')]),1281 [(self._create_address('c'), None)],1282 "Address 'c' has already been deleted from state.")1283 self.context_manager.delete(ctx_3b, [self._create_address('b')])1284 ctx_4 = self.context_manager.create_context(1285 state_hash=sh1,1286 base_contexts=[ctx_3b, ctx_3a],1287 inputs=[""],1288 outputs=[""])1289 self.assertEqual(1290 self.context_manager.get(ctx_4, [self._create_address('b'),1291 self._create_address('c'),1292 self._create_address('d')]),1293 [(self._create_address('b'), None),1294 (self._create_address('c'), None),1295 (self._create_address('d'), None)],1296 "Addresses 'b', 'c', and 'd' have been deleted from state.")1297 sh2 = squash(1298 state_root=sh1,1299 context_ids=[ctx_4],1300 persist=True,1301 clean_up=True)1302 tree = MerkleDatabase(self.database_results)1303 sh1_assertion = tree.update(1304 {1305 self._create_address('b'): b'1',1306 self._create_address('c'): b'2',1307 self._create_address('d'): b'3'1308 },1309 virtual=False)1310 self.assertEqual(sh1, sh1_assertion,1311 "The middle state hash must be correct.")1312 tree.set_merkle_root(sh1)1313 sh2_assertion = tree.update(1314 {1315 self._create_address('e'): b'2'1316 },1317 delete_items=[1318 self._create_address('b'),1319 self._create_address('c'),1320 self._create_address('d')1321 ],1322 virtual=False)1323 self.assertEqual(sh2, sh2_assertion,...

Full Screen

Full Screen

test_schedulers_with_yaml.py

Source:test_schedulers_with_yaml.py Github

copy

Full Screen

1# Copyright 2017 Intel Corporation2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14# ----------------------------------------------------------------------------15import unittest16import logging17import os18import shutil19import tempfile20from sawtooth_validator.database.native_lmdb import NativeLmdbDatabase21from sawtooth_validator.state.merkle import MerkleDatabase22from sawtooth_validator.execution.context_manager import ContextManager23from sawtooth_validator.execution.scheduler_parallel import ParallelScheduler24from sawtooth_validator.execution.scheduler_serial import SerialScheduler25from test_scheduler.yaml_scheduler_tester import SchedulerTester26LOGGER = logging.getLogger(__name__)27class TestSchedulersWithYaml(unittest.TestCase):28 def __init__(self, test_name):29 super().__init__(test_name)30 self._temp_dir = None31 def setUp(self):32 self._temp_dir = tempfile.mkdtemp()33 database = NativeLmdbDatabase(34 os.path.join(self._temp_dir, 'test_state_view.lmdb'),35 indexes=MerkleDatabase.create_index_configuration(),36 _size=10 * 1024 * 1024)37 self._context_manager = ContextManager(database)38 def tearDown(self):39 self._context_manager.stop()40 shutil.rmtree(self._temp_dir)41 def _setup_serial_scheduler(self):42 context_manager = self._context_manager43 squash_handler = context_manager.get_squash_handler()44 first_state_root = context_manager.get_first_root()45 scheduler = SerialScheduler(squash_handler,46 first_state_root,47 always_persist=False)48 return context_manager, scheduler49 def _setup_parallel_scheduler(self):50 context_manager = self._context_manager51 squash_handler = context_manager.get_squash_handler()52 first_state_root = context_manager.get_first_root()53 scheduler = ParallelScheduler(squash_handler,54 first_state_root,55 always_persist=False)56 return context_manager, scheduler57 def _get_filenames(self):58 base_dir = os.path.abspath(os.path.dirname(__file__))59 data_dir = os.path.join(base_dir, 'data')60 filepaths = []61 for root, _, filenames in os.walk(data_dir):62 filepaths.extend(63 map(64 lambda f, r=root: os.path.join(r, f),65 filter(66 lambda f: f.endswith('.yml') or f.endswith('.yaml'),67 filenames)))68 return filepaths69 def test_all_yaml_files(self):70 """Tests the schedulers against each of the yaml files in the71 test_scheduler/data directory"""72 for file_path in self._get_filenames():73 try:74 context_manager, scheduler = self._setup_parallel_scheduler()75 self._single_block_files_individually(76 scheduler=scheduler,77 context_manager=context_manager,78 name=file_path)79 context_manager, scheduler = self._setup_parallel_scheduler()80 self._single_block_files_individually(81 scheduler=scheduler,82 context_manager=context_manager,83 name=file_path,84 lifo=True)85 context_manager, scheduler = self._setup_parallel_scheduler()86 self._single_block_files_individually_alt(87 scheduler=scheduler,88 context_manager=context_manager,89 name=file_path)90 context_manager, scheduler = self._setup_parallel_scheduler()91 self._single_block_files_individually_alt(92 scheduler=scheduler,93 context_manager=context_manager,94 name=file_path,95 lifo=True)96 context_manager, scheduler = self._setup_serial_scheduler()97 self._single_block_files_individually(98 scheduler=scheduler,99 context_manager=context_manager,100 name=file_path)101 context_manager, scheduler = self._setup_serial_scheduler()102 self._single_block_files_individually_alt(103 scheduler=scheduler,104 context_manager=context_manager,105 name=file_path)106 except AssertionError:107 LOGGER.warning("Failure on file %s", file_path)108 raise109 def _single_block_files_individually_alt(self,110 scheduler,111 context_manager,112 name,113 lifo=False):114 """Tests scheduler(s) with yaml files that represent a single115 block.116 Notes:117 Tests that the serial scheduler has the correct batch validity118 and state hash, and that 1 state hash is produced.119 """120 file_name = name121 tester = SchedulerTester(file_name)122 defined_batch_results_dict = tester.batch_results123 batch_results, txns_to_assert_state = tester.run_scheduler_alternating(124 scheduler=scheduler,125 context_manager=context_manager,126 txns_executed_fifo=not lifo)127 self._assertions(tester, defined_batch_results_dict,128 batch_results,129 txns_to_assert_state, name)130 def _single_block_files_individually(self,131 scheduler,132 context_manager,133 name,134 lifo=False):135 """Tests scheduler(s) with yaml files that represent a single136 block.137 Notes:138 Tests that the serial scheduler has the correct batch validity139 and state hash, and that 1 state hash is produced.140 """141 file_name = name142 tester = SchedulerTester(file_name)143 defined_batch_results_dict = tester.batch_results144 batch_results, txns_to_assert_state = tester.run_scheduler(145 scheduler=scheduler,146 context_manager=context_manager,147 txns_executed_fifo=not lifo)148 self._assertions(tester, defined_batch_results_dict,149 batch_results,150 txns_to_assert_state, name)151 def _assertions(self, tester, defined_batch_results_dict,152 batch_results, txns_to_assert_state, name):153 self.assert_batch_validity(154 defined_batch_results_dict,155 batch_results)156 for t_id in txns_to_assert_state:157 state_and_txn_context = txns_to_assert_state.get(t_id)158 if state_and_txn_context is not None:159 txn_context, state_found, state_assert = state_and_txn_context160 self.assertEqual(state_found, state_assert,161 "Transaction {} in batch {} has the wrong "162 "state in the context".format(163 txn_context.txn_num,164 txn_context.batch_num))165 self.assert_one_state_hash(batch_results=batch_results)166 sched_state_roots = self._get_state_roots(167 batch_results=batch_results)168 calc_state_hash = tester.compute_state_hashes_wo_scheduler(169 self._temp_dir)170 self.assertEqual(171 sched_state_roots,172 calc_state_hash,173 "The state hashes calculated by the scheduler for yaml file {}"174 " must be the same as calculated by the tester".format(name))175 def _get_state_roots(self, batch_results):176 return [r.state_hash for _, r in batch_results177 if r.state_hash is not None]178 def assert_batch_validity(self, yaml_results_dict, batch_results):179 """Checks that all of the BatchExecutionResults calculated are the180 same as defined in the yaml file and returned by181 SchedulerTester.create_batches().182 Args:183 yaml_results_dict (dict): batch signature: BatchExecutionResult,184 Calculated from the yaml.185 batch_results (list): list of tuples186 (batch signature, BatchExecutionResult)187 Calculated by the scheduler.188 Raises:189 AssertionError190 """191 for i, b_e in enumerate(batch_results):192 signature, result = b_e193 defined_result = yaml_results_dict[signature]194 self.assertIsNotNone(result.is_valid, "Batch #{} has None as it's "195 "validity".format(i))196 self.assertEqual(197 result.is_valid,198 defined_result.is_valid,199 "Batch #{} was defined in the yaml to be {},"200 "but the scheduler determined "201 "it was {}".format(202 i + 1,203 'valid' if defined_result.is_valid else 'invalid',204 'valid' if result.is_valid else 'invalid'))205 def assert_one_state_hash(self, batch_results):206 """Should be used when only one state hash is expected from the207 scheduler.208 Args:209 batch_results (list): list of tuples210 (batch signature, BatchExecutionResult)211 Raises:212 AssertionError213 """214 state_roots = self._get_state_roots(batch_results=batch_results)215 self.assertEqual(len(state_roots), 1,216 "The scheduler calculated more than one state "...

Full Screen

Full Screen

batch_thompson.py

Source:batch_thompson.py Github

copy

Full Screen

1# Copyright (c) 2016, the GPyOpt Authors2# Licensed under the BSD 3-clause license (see LICENSE.txt)3from .base import SamplingBasedBatchEvaluator4from ...optimization.anchor_points_generator import ThompsonSamplingAnchorPointsGenerator5from ...optimization.optimizer import OptLbfgs, apply_optimizer, choose_optimizer6import numpy as np7class ThompsonBatch(SamplingBasedBatchEvaluator):8 """9 Class for a Thompson batch method. Elements are selected iteratively using the current acquistion function but exploring the models10 by using Thompson sampling11 :param acquisition: acquisition function to be used to compute the batch.12 :param batch size: the number of elements in the batch.13 """14 def __init__(self, acquisition, batch_size):15 super(ThompsonBatch, self).__init__(acquisition, batch_size)16 self.model = self.acquisition.model17 self.optimizer_name = 'lbfgs'18 self.f = self.acquisition.acquisition_function19 self.f_df = self.acquisition.acquisition_function_withGradients20 self.space = self.acquisition.space21 def initialize_batch(self, duplicate_manager=None, context_manager=None):22 return None23 def get_anchor_points(self, duplicate_manager=None, context_manager=None):24 design_type, unique = "random", False25 if duplicate_manager:26 unique = True27 anchor_points_generator = ThompsonSamplingAnchorPointsGenerator(self.space, design_type, model=self.model)28 return anchor_points_generator.get(num_anchor=self.num_anchor, duplicate_manager=duplicate_manager, unique=unique, context_manager = self.context_manager)29 def optimize_anchor_point(self, a, duplicate_manager=None, context_manager=None):30 ### --- Update the bounds of the default optimizer according to the context_manager31 if context_manager:32 bounds = self.context_manager.noncontext_bounds33 else:34 bounds = self.space.get_bounds()35 self.local_optimizer = choose_optimizer(self.optimizer_name, bounds)36 ### --- Run the local optimizer37 x, _ = apply_optimizer(self.local_optimizer, a, f=self.f, df=None, f_df=self.f_df, duplicate_manager=duplicate_manager, context_manager = self.context_manager, space=self.space)38 return self.space.round_optimum(x)39 def compute_batch_without_duplicate_logic(self, context_manager=None):40 anchor_points = self.get_anchor_points(context_manager=context_manager)...

Full Screen

Full Screen

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.

LambdaTest Learning Hubs:

YouTube

You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.

Run pytest-mock automation tests on LambdaTest cloud grid

Perform automation testing on 3000+ real desktop and mobile devices online.

Try LambdaTest Now !!

Get 100 minutes of automation test minutes FREE!!

Next-Gen App & Browser Testing Cloud

Was this article helpful?

Helpful

NotHelpful