Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More
How to use put_object method in localstack
Best Python code snippet using localstack_python
test_backend.py
Source:test_backend.py 
...49 # Test ContainerBroker.empty50 broker = ContainerBroker(':memory:', account='a', container='c')51 broker.initialize(normalize_timestamp('1'))52 self.assert_(broker.empty())53 broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain',54 'd41d8cd98f00b204e9800998ecf8427e')55 self.assert_(not broker.empty())56 sleep(.00001)57 broker.delete_object('o', normalize_timestamp(time()))58 self.assert_(broker.empty())59 def test_reclaim(self):60 broker = ContainerBroker(':memory:', account='test_account',61 container='test_container')62 broker.initialize(normalize_timestamp('1'))63 broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain',64 'd41d8cd98f00b204e9800998ecf8427e')65 with broker.get() as conn:66 self.assertEquals(conn.execute(67 "SELECT count(*) FROM object "68 "WHERE deleted = 0").fetchone()[0], 1)69 self.assertEquals(conn.execute(70 "SELECT count(*) FROM object "71 "WHERE deleted = 1").fetchone()[0], 0)72 broker.reclaim(normalize_timestamp(time() - 999), time())73 with broker.get() as conn:74 self.assertEquals(conn.execute(75 "SELECT count(*) FROM object "76 "WHERE deleted = 0").fetchone()[0], 1)77 self.assertEquals(conn.execute(78 "SELECT count(*) FROM object "79 "WHERE deleted = 1").fetchone()[0], 0)80 sleep(.00001)81 broker.delete_object('o', normalize_timestamp(time()))82 with broker.get() as conn:83 self.assertEquals(conn.execute(84 "SELECT count(*) FROM object "85 "WHERE deleted = 0").fetchone()[0], 0)86 self.assertEquals(conn.execute(87 "SELECT count(*) FROM object "88 "WHERE deleted = 1").fetchone()[0], 1)89 broker.reclaim(normalize_timestamp(time() - 999), time())90 with broker.get() as conn:91 self.assertEquals(conn.execute(92 "SELECT count(*) FROM object "93 "WHERE deleted = 0").fetchone()[0], 0)94 self.assertEquals(conn.execute(95 "SELECT count(*) FROM object "96 "WHERE deleted = 1").fetchone()[0], 1)97 sleep(.00001)98 broker.reclaim(normalize_timestamp(time()), time())99 with broker.get() as conn:100 self.assertEquals(conn.execute(101 "SELECT count(*) FROM object "102 "WHERE deleted = 0").fetchone()[0], 0)103 self.assertEquals(conn.execute(104 "SELECT count(*) FROM object "105 "WHERE deleted = 1").fetchone()[0], 0)106 # Test the return values of reclaim()107 broker.put_object('w', normalize_timestamp(time()), 0, 'text/plain',108 'd41d8cd98f00b204e9800998ecf8427e')109 broker.put_object('x', normalize_timestamp(time()), 0, 'text/plain',110 'd41d8cd98f00b204e9800998ecf8427e')111 broker.put_object('y', normalize_timestamp(time()), 0, 'text/plain',112 'd41d8cd98f00b204e9800998ecf8427e')113 broker.put_object('z', normalize_timestamp(time()), 0, 'text/plain',114 'd41d8cd98f00b204e9800998ecf8427e')115 # Test before deletion116 broker.reclaim(normalize_timestamp(time()), time())117 broker.delete_db(normalize_timestamp(time()))118 def test_delete_object(self):119 # Test ContainerBroker.delete_object120 broker = ContainerBroker(':memory:', account='a', container='c')121 broker.initialize(normalize_timestamp('1'))122 broker.put_object('o', normalize_timestamp(time()), 0, 'text/plain',123 'd41d8cd98f00b204e9800998ecf8427e')124 with broker.get() as conn:125 self.assertEquals(conn.execute(126 "SELECT count(*) FROM object "127 "WHERE deleted = 0").fetchone()[0], 1)128 self.assertEquals(conn.execute(129 "SELECT count(*) FROM object "130 "WHERE deleted = 1").fetchone()[0], 0)131 sleep(.00001)132 broker.delete_object('o', normalize_timestamp(time()))133 with broker.get() as conn:134 self.assertEquals(conn.execute(135 "SELECT count(*) FROM object "136 "WHERE deleted = 0").fetchone()[0], 0)137 self.assertEquals(conn.execute(138 "SELECT count(*) FROM object "139 "WHERE deleted = 1").fetchone()[0], 1)140 def test_put_object(self):141 # Test ContainerBroker.put_object142 broker = ContainerBroker(':memory:', account='a', container='c')143 broker.initialize(normalize_timestamp('1'))144 # Create initial object145 timestamp = normalize_timestamp(time())146 broker.put_object('"{<object \'&\' name>}"', timestamp, 123,147 'application/x-test',148 '5af83e3196bf99f440f31f2e1a6c9afe')149 with broker.get() as conn:150 self.assertEquals(conn.execute(151 "SELECT name FROM object").fetchone()[0],152 '"{<object \'&\' name>}"')153 self.assertEquals(conn.execute(154 "SELECT created_at FROM object").fetchone()[0], timestamp)155 self.assertEquals(conn.execute(156 "SELECT size FROM object").fetchone()[0], 123)157 self.assertEquals(conn.execute(158 "SELECT content_type FROM object").fetchone()[0],159 'application/x-test')160 self.assertEquals(conn.execute(161 "SELECT etag FROM object").fetchone()[0],162 '5af83e3196bf99f440f31f2e1a6c9afe')163 self.assertEquals(conn.execute(164 "SELECT deleted FROM object").fetchone()[0], 0)165 # Reput same event166 broker.put_object('"{<object \'&\' name>}"', timestamp, 123,167 'application/x-test',168 '5af83e3196bf99f440f31f2e1a6c9afe')169 with broker.get() as conn:170 self.assertEquals(conn.execute(171 "SELECT name FROM object").fetchone()[0],172 '"{<object \'&\' name>}"')173 self.assertEquals(conn.execute(174 "SELECT created_at FROM object").fetchone()[0], timestamp)175 self.assertEquals(conn.execute(176 "SELECT size FROM object").fetchone()[0], 123)177 self.assertEquals(conn.execute(178 "SELECT content_type FROM object").fetchone()[0],179 'application/x-test')180 self.assertEquals(conn.execute(181 "SELECT etag FROM object").fetchone()[0],182 '5af83e3196bf99f440f31f2e1a6c9afe')183 self.assertEquals(conn.execute(184 "SELECT deleted FROM object").fetchone()[0], 0)185 # Put new event186 sleep(.00001)187 timestamp = normalize_timestamp(time())188 broker.put_object('"{<object \'&\' name>}"', timestamp, 124,189 'application/x-test',190 'aa0749bacbc79ec65fe206943d8fe449')191 with broker.get() as conn:192 self.assertEquals(conn.execute(193 "SELECT name FROM object").fetchone()[0],194 '"{<object \'&\' name>}"')195 self.assertEquals(conn.execute(196 "SELECT created_at FROM object").fetchone()[0], timestamp)197 self.assertEquals(conn.execute(198 "SELECT size FROM object").fetchone()[0], 124)199 self.assertEquals(conn.execute(200 "SELECT content_type FROM object").fetchone()[0],201 'application/x-test')202 self.assertEquals(conn.execute(203 "SELECT etag FROM object").fetchone()[0],204 'aa0749bacbc79ec65fe206943d8fe449')205 self.assertEquals(conn.execute(206 "SELECT deleted FROM object").fetchone()[0], 0)207 # Put old event208 otimestamp = normalize_timestamp(float(timestamp) - 1)209 broker.put_object('"{<object \'&\' name>}"', otimestamp, 124,210 'application/x-test',211 'aa0749bacbc79ec65fe206943d8fe449')212 with broker.get() as conn:213 self.assertEquals(conn.execute(214 "SELECT name FROM object").fetchone()[0],215 '"{<object \'&\' name>}"')216 self.assertEquals(conn.execute(217 "SELECT created_at FROM object").fetchone()[0], timestamp)218 self.assertEquals(conn.execute(219 "SELECT size FROM object").fetchone()[0], 124)220 self.assertEquals(conn.execute(221 "SELECT content_type FROM object").fetchone()[0],222 'application/x-test')223 self.assertEquals(conn.execute(224 "SELECT etag FROM object").fetchone()[0],225 'aa0749bacbc79ec65fe206943d8fe449')226 self.assertEquals(conn.execute(227 "SELECT deleted FROM object").fetchone()[0], 0)228 # Put old delete event229 dtimestamp = normalize_timestamp(float(timestamp) - 1)230 broker.put_object('"{<object \'&\' name>}"', dtimestamp, 0, '', '',231 deleted=1)232 with broker.get() as conn:233 self.assertEquals(conn.execute(234 "SELECT name FROM object").fetchone()[0],235 '"{<object \'&\' name>}"')236 self.assertEquals(conn.execute(237 "SELECT created_at FROM object").fetchone()[0], timestamp)238 self.assertEquals(conn.execute(239 "SELECT size FROM object").fetchone()[0], 124)240 self.assertEquals(conn.execute(241 "SELECT content_type FROM object").fetchone()[0],242 'application/x-test')243 self.assertEquals(conn.execute(244 "SELECT etag FROM object").fetchone()[0],245 'aa0749bacbc79ec65fe206943d8fe449')246 self.assertEquals(conn.execute(247 "SELECT deleted FROM object").fetchone()[0], 0)248 # Put new delete event249 sleep(.00001)250 timestamp = normalize_timestamp(time())251 broker.put_object('"{<object \'&\' name>}"', timestamp, 0, '', '',252 deleted=1)253 with broker.get() as conn:254 self.assertEquals(conn.execute(255 "SELECT name FROM object").fetchone()[0],256 '"{<object \'&\' name>}"')257 self.assertEquals(conn.execute(258 "SELECT created_at FROM object").fetchone()[0], timestamp)259 self.assertEquals(conn.execute(260 "SELECT deleted FROM object").fetchone()[0], 1)261 # Put new event262 sleep(.00001)263 timestamp = normalize_timestamp(time())264 broker.put_object('"{<object \'&\' name>}"', timestamp, 123,265 'application/x-test',266 '5af83e3196bf99f440f31f2e1a6c9afe')267 with broker.get() as conn:268 self.assertEquals(conn.execute(269 "SELECT name FROM object").fetchone()[0],270 '"{<object \'&\' name>}"')271 self.assertEquals(conn.execute(272 "SELECT created_at FROM object").fetchone()[0], timestamp)273 self.assertEquals(conn.execute(274 "SELECT size FROM object").fetchone()[0], 123)275 self.assertEquals(conn.execute(276 "SELECT content_type FROM object").fetchone()[0],277 'application/x-test')278 self.assertEquals(conn.execute(279 "SELECT etag FROM object").fetchone()[0],280 '5af83e3196bf99f440f31f2e1a6c9afe')281 self.assertEquals(conn.execute(282 "SELECT deleted FROM object").fetchone()[0], 0)283 # We'll use this later284 sleep(.0001)285 in_between_timestamp = normalize_timestamp(time())286 # New post event287 sleep(.0001)288 previous_timestamp = timestamp289 timestamp = normalize_timestamp(time())290 with broker.get() as conn:291 self.assertEquals(conn.execute(292 "SELECT name FROM object").fetchone()[0],293 '"{<object \'&\' name>}"')294 self.assertEquals(conn.execute(295 "SELECT created_at FROM object").fetchone()[0],296 previous_timestamp)297 self.assertEquals(conn.execute(298 "SELECT size FROM object").fetchone()[0], 123)299 self.assertEquals(conn.execute(300 "SELECT content_type FROM object").fetchone()[0],301 'application/x-test')302 self.assertEquals(conn.execute(303 "SELECT etag FROM object").fetchone()[0],304 '5af83e3196bf99f440f31f2e1a6c9afe')305 self.assertEquals(conn.execute(306 "SELECT deleted FROM object").fetchone()[0], 0)307 # Put event from after last put but before last post308 timestamp = in_between_timestamp309 broker.put_object('"{<object \'&\' name>}"', timestamp, 456,310 'application/x-test3',311 '6af83e3196bf99f440f31f2e1a6c9afe')312 with broker.get() as conn:313 self.assertEquals(conn.execute(314 "SELECT name FROM object").fetchone()[0],315 '"{<object \'&\' name>}"')316 self.assertEquals(conn.execute(317 "SELECT created_at FROM object").fetchone()[0], timestamp)318 self.assertEquals(conn.execute(319 "SELECT size FROM object").fetchone()[0], 456)320 self.assertEquals(conn.execute(321 "SELECT content_type FROM object").fetchone()[0],322 'application/x-test3')323 self.assertEquals(conn.execute(324 "SELECT etag FROM object").fetchone()[0],325 '6af83e3196bf99f440f31f2e1a6c9afe')326 self.assertEquals(conn.execute(327 "SELECT deleted FROM object").fetchone()[0], 0)328 def test_get_info(self):329 # Test ContainerBroker.get_info330 broker = ContainerBroker(':memory:', account='test1',331 container='test2')332 broker.initialize(normalize_timestamp('1'))333 info = broker.get_info()334 self.assertEquals(info['account'], 'test1')335 self.assertEquals(info['container'], 'test2')336 self.assertEquals(info['hash'], '00000000000000000000000000000000')337 info = broker.get_info()338 self.assertEquals(info['object_count'], 0)339 self.assertEquals(info['bytes_used'], 0)340 broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain',341 '5af83e3196bf99f440f31f2e1a6c9afe')342 info = broker.get_info()343 self.assertEquals(info['object_count'], 1)344 self.assertEquals(info['bytes_used'], 123)345 sleep(.00001)346 broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain',347 '5af83e3196bf99f440f31f2e1a6c9afe')348 info = broker.get_info()349 self.assertEquals(info['object_count'], 2)350 self.assertEquals(info['bytes_used'], 246)351 sleep(.00001)352 broker.put_object('o2', normalize_timestamp(time()), 1000,353 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe')354 info = broker.get_info()355 self.assertEquals(info['object_count'], 2)356 self.assertEquals(info['bytes_used'], 1123)357 sleep(.00001)358 broker.delete_object('o1', normalize_timestamp(time()))359 info = broker.get_info()360 self.assertEquals(info['object_count'], 1)361 self.assertEquals(info['bytes_used'], 1000)362 sleep(.00001)363 broker.delete_object('o2', normalize_timestamp(time()))364 info = broker.get_info()365 self.assertEquals(info['object_count'], 0)366 self.assertEquals(info['bytes_used'], 0)367 info = broker.get_info()368 self.assertEquals(info['x_container_sync_point1'], -1)369 self.assertEquals(info['x_container_sync_point2'], -1)370 def test_set_x_syncs(self):371 broker = ContainerBroker(':memory:', account='test1',372 container='test2')373 broker.initialize(normalize_timestamp('1'))374 info = broker.get_info()375 self.assertEquals(info['x_container_sync_point1'], -1)376 self.assertEquals(info['x_container_sync_point2'], -1)377 broker.set_x_container_sync_points(1, 2)378 info = broker.get_info()379 self.assertEquals(info['x_container_sync_point1'], 1)380 self.assertEquals(info['x_container_sync_point2'], 2)381 def test_get_report_info(self):382 broker = ContainerBroker(':memory:', account='test1',383 container='test2')384 broker.initialize(normalize_timestamp('1'))385 info = broker.get_info()386 self.assertEquals(info['account'], 'test1')387 self.assertEquals(info['container'], 'test2')388 self.assertEquals(info['object_count'], 0)389 self.assertEquals(info['bytes_used'], 0)390 self.assertEquals(info['reported_object_count'], 0)391 self.assertEquals(info['reported_bytes_used'], 0)392 broker.put_object('o1', normalize_timestamp(time()), 123, 'text/plain',393 '5af83e3196bf99f440f31f2e1a6c9afe')394 info = broker.get_info()395 self.assertEquals(info['object_count'], 1)396 self.assertEquals(info['bytes_used'], 123)397 self.assertEquals(info['reported_object_count'], 0)398 self.assertEquals(info['reported_bytes_used'], 0)399 sleep(.00001)400 broker.put_object('o2', normalize_timestamp(time()), 123, 'text/plain',401 '5af83e3196bf99f440f31f2e1a6c9afe')402 info = broker.get_info()403 self.assertEquals(info['object_count'], 2)404 self.assertEquals(info['bytes_used'], 246)405 self.assertEquals(info['reported_object_count'], 0)406 self.assertEquals(info['reported_bytes_used'], 0)407 sleep(.00001)408 broker.put_object('o2', normalize_timestamp(time()), 1000,409 'text/plain', '5af83e3196bf99f440f31f2e1a6c9afe')410 info = broker.get_info()411 self.assertEquals(info['object_count'], 2)412 self.assertEquals(info['bytes_used'], 1123)413 self.assertEquals(info['reported_object_count'], 0)414 self.assertEquals(info['reported_bytes_used'], 0)415 put_timestamp = normalize_timestamp(time())416 sleep(.001)417 delete_timestamp = normalize_timestamp(time())418 broker.reported(put_timestamp, delete_timestamp, 2, 1123)419 info = broker.get_info()420 self.assertEquals(info['object_count'], 2)421 self.assertEquals(info['bytes_used'], 1123)422 self.assertEquals(info['reported_put_timestamp'], put_timestamp)423 self.assertEquals(info['reported_delete_timestamp'], delete_timestamp)424 self.assertEquals(info['reported_object_count'], 2)425 self.assertEquals(info['reported_bytes_used'], 1123)426 sleep(.00001)427 broker.delete_object('o1', normalize_timestamp(time()))428 info = broker.get_info()429 self.assertEquals(info['object_count'], 1)430 self.assertEquals(info['bytes_used'], 1000)431 self.assertEquals(info['reported_object_count'], 2)432 self.assertEquals(info['reported_bytes_used'], 1123)433 sleep(.00001)434 broker.delete_object('o2', normalize_timestamp(time()))435 info = broker.get_info()436 self.assertEquals(info['object_count'], 0)437 self.assertEquals(info['bytes_used'], 0)438 self.assertEquals(info['reported_object_count'], 2)439 self.assertEquals(info['reported_bytes_used'], 1123)440 def test_list_objects_iter(self):441 # Test ContainerBroker.list_objects_iter442 broker = ContainerBroker(':memory:', account='a', container='c')443 broker.initialize(normalize_timestamp('1'))444 for obj1 in xrange(4):445 for obj2 in xrange(125):446 broker.put_object('%d/%04d' % (obj1, obj2),447 normalize_timestamp(time()), 0, 'text/plain',448 'd41d8cd98f00b204e9800998ecf8427e')449 for obj in xrange(125):450 broker.put_object('2/0051/%04d' % obj,451 normalize_timestamp(time()), 0, 'text/plain',452 'd41d8cd98f00b204e9800998ecf8427e')453 for obj in xrange(125):454 broker.put_object('3/%04d/0049' % obj,455 normalize_timestamp(time()), 0, 'text/plain',456 'd41d8cd98f00b204e9800998ecf8427e')457 listing = broker.list_objects_iter(100, '', None, None, '')458 self.assertEquals(len(listing), 100)459 self.assertEquals(listing[0][0], '0/0000')460 self.assertEquals(listing[-1][0], '0/0099')461 listing = broker.list_objects_iter(100, '', '0/0050', None, '')462 self.assertEquals(len(listing), 50)463 self.assertEquals(listing[0][0], '0/0000')464 self.assertEquals(listing[-1][0], '0/0049')465 listing = broker.list_objects_iter(100, '0/0099', None, None, '')466 self.assertEquals(len(listing), 100)467 self.assertEquals(listing[0][0], '0/0100')468 self.assertEquals(listing[-1][0], '1/0074')469 listing = broker.list_objects_iter(55, '1/0074', None, None, '')470 self.assertEquals(len(listing), 55)471 self.assertEquals(listing[0][0], '1/0075')472 self.assertEquals(listing[-1][0], '2/0004')473 listing = broker.list_objects_iter(10, '', None, '0/01', '')474 self.assertEquals(len(listing), 10)475 self.assertEquals(listing[0][0], '0/0100')476 self.assertEquals(listing[-1][0], '0/0109')477 listing = broker.list_objects_iter(10, '', None, '0/', '/')478 self.assertEquals(len(listing), 10)479 self.assertEquals(listing[0][0], '0/0000')480 self.assertEquals(listing[-1][0], '0/0009')481 # Same as above, but using the path argument.482 listing = broker.list_objects_iter(10, '', None, None, '', '0')483 self.assertEquals(len(listing), 10)484 self.assertEquals(listing[0][0], '0/0000')485 self.assertEquals(listing[-1][0], '0/0009')486 listing = broker.list_objects_iter(10, '', None, '', '/')487 self.assertEquals(len(listing), 4)488 self.assertEquals([row[0] for row in listing],489 ['0/', '1/', '2/', '3/'])490 listing = broker.list_objects_iter(10, '2', None, None, '/')491 self.assertEquals(len(listing), 2)492 self.assertEquals([row[0] for row in listing], ['2/', '3/'])493 listing = broker.list_objects_iter(10, '2/', None, None, '/')494 self.assertEquals(len(listing), 1)495 self.assertEquals([row[0] for row in listing], ['3/'])496 listing = broker.list_objects_iter(10, '2/0050', None, '2/', '/')497 self.assertEquals(len(listing), 10)498 self.assertEquals(listing[0][0], '2/0051')499 self.assertEquals(listing[1][0], '2/0051/')500 self.assertEquals(listing[2][0], '2/0052')501 self.assertEquals(listing[-1][0], '2/0059')502 listing = broker.list_objects_iter(10, '3/0045', None, '3/', '/')503 self.assertEquals(len(listing), 10)504 self.assertEquals([row[0] for row in listing],505 ['3/0045/', '3/0046', '3/0046/', '3/0047',506 '3/0047/', '3/0048', '3/0048/', '3/0049',507 '3/0049/', '3/0050'])508 broker.put_object('3/0049/', normalize_timestamp(time()), 0,509 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')510 listing = broker.list_objects_iter(10, '3/0048', None, None, None)511 self.assertEquals(len(listing), 10)512 self.assertEquals(513 [row[0] for row in listing],514 ['3/0048/0049', '3/0049', '3/0049/',515 '3/0049/0049', '3/0050', '3/0050/0049', '3/0051', '3/0051/0049',516 '3/0052', '3/0052/0049'])517 listing = broker.list_objects_iter(10, '3/0048', None, '3/', '/')518 self.assertEquals(len(listing), 10)519 self.assertEquals(520 [row[0] for row in listing],521 ['3/0048/', '3/0049', '3/0049/', '3/0050',522 '3/0050/', '3/0051', '3/0051/', '3/0052', '3/0052/', '3/0053'])523 listing = broker.list_objects_iter(10, None, None, '3/0049/', '/')524 self.assertEquals(len(listing), 2)525 self.assertEquals(526 [row[0] for row in listing],527 ['3/0049/', '3/0049/0049'])528 listing = broker.list_objects_iter(10, None, None, None, None,529 '3/0049')530 self.assertEquals(len(listing), 1)531 self.assertEquals([row[0] for row in listing], ['3/0049/0049'])532 listing = broker.list_objects_iter(2, None, None, '3/', '/')533 self.assertEquals(len(listing), 2)534 self.assertEquals([row[0] for row in listing], ['3/0000', '3/0000/'])535 listing = broker.list_objects_iter(2, None, None, None, None, '3')536 self.assertEquals(len(listing), 2)537 self.assertEquals([row[0] for row in listing], ['3/0000', '3/0001'])538 def test_list_objects_iter_non_slash(self):539 # Test ContainerBroker.list_objects_iter using a540 # delimiter that is not a slash541 broker = ContainerBroker(':memory:', account='a', container='c')542 broker.initialize(normalize_timestamp('1'))543 for obj1 in xrange(4):544 for obj2 in xrange(125):545 broker.put_object('%d:%04d' % (obj1, obj2),546 normalize_timestamp(time()), 0, 'text/plain',547 'd41d8cd98f00b204e9800998ecf8427e')548 for obj in xrange(125):549 broker.put_object('2:0051:%04d' % obj,550 normalize_timestamp(time()), 0, 'text/plain',551 'd41d8cd98f00b204e9800998ecf8427e')552 for obj in xrange(125):553 broker.put_object('3:%04d:0049' % obj,554 normalize_timestamp(time()), 0, 'text/plain',555 'd41d8cd98f00b204e9800998ecf8427e')556 listing = broker.list_objects_iter(100, '', None, None, '')557 self.assertEquals(len(listing), 100)558 self.assertEquals(listing[0][0], '0:0000')559 self.assertEquals(listing[-1][0], '0:0099')560 listing = broker.list_objects_iter(100, '', '0:0050', None, '')561 self.assertEquals(len(listing), 50)562 self.assertEquals(listing[0][0], '0:0000')563 self.assertEquals(listing[-1][0], '0:0049')564 listing = broker.list_objects_iter(100, '0:0099', None, None, '')565 self.assertEquals(len(listing), 100)566 self.assertEquals(listing[0][0], '0:0100')567 self.assertEquals(listing[-1][0], '1:0074')568 listing = broker.list_objects_iter(55, '1:0074', None, None, '')569 self.assertEquals(len(listing), 55)570 self.assertEquals(listing[0][0], '1:0075')571 self.assertEquals(listing[-1][0], '2:0004')572 listing = broker.list_objects_iter(10, '', None, '0:01', '')573 self.assertEquals(len(listing), 10)574 self.assertEquals(listing[0][0], '0:0100')575 self.assertEquals(listing[-1][0], '0:0109')576 listing = broker.list_objects_iter(10, '', None, '0:', ':')577 self.assertEquals(len(listing), 10)578 self.assertEquals(listing[0][0], '0:0000')579 self.assertEquals(listing[-1][0], '0:0009')580 # Same as above, but using the path argument, so nothing should be581 # returned since path uses a '/' as a delimiter.582 listing = broker.list_objects_iter(10, '', None, None, '', '0')583 self.assertEquals(len(listing), 0)584 listing = broker.list_objects_iter(10, '', None, '', ':')585 self.assertEquals(len(listing), 4)586 self.assertEquals([row[0] for row in listing],587 ['0:', '1:', '2:', '3:'])588 listing = broker.list_objects_iter(10, '2', None, None, ':')589 self.assertEquals(len(listing), 2)590 self.assertEquals([row[0] for row in listing], ['2:', '3:'])591 listing = broker.list_objects_iter(10, '2:', None, None, ':')592 self.assertEquals(len(listing), 1)593 self.assertEquals([row[0] for row in listing], ['3:'])594 listing = broker.list_objects_iter(10, '2:0050', None, '2:', ':')595 self.assertEquals(len(listing), 10)596 self.assertEquals(listing[0][0], '2:0051')597 self.assertEquals(listing[1][0], '2:0051:')598 self.assertEquals(listing[2][0], '2:0052')599 self.assertEquals(listing[-1][0], '2:0059')600 listing = broker.list_objects_iter(10, '3:0045', None, '3:', ':')601 self.assertEquals(len(listing), 10)602 self.assertEquals([row[0] for row in listing],603 ['3:0045:', '3:0046', '3:0046:', '3:0047',604 '3:0047:', '3:0048', '3:0048:', '3:0049',605 '3:0049:', '3:0050'])606 broker.put_object('3:0049:', normalize_timestamp(time()), 0,607 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')608 listing = broker.list_objects_iter(10, '3:0048', None, None, None)609 self.assertEquals(len(listing), 10)610 self.assertEquals(611 [row[0] for row in listing],612 ['3:0048:0049', '3:0049', '3:0049:',613 '3:0049:0049', '3:0050', '3:0050:0049', '3:0051', '3:0051:0049',614 '3:0052', '3:0052:0049'])615 listing = broker.list_objects_iter(10, '3:0048', None, '3:', ':')616 self.assertEquals(len(listing), 10)617 self.assertEquals(618 [row[0] for row in listing],619 ['3:0048:', '3:0049', '3:0049:', '3:0050',620 '3:0050:', '3:0051', '3:0051:', '3:0052', '3:0052:', '3:0053'])621 listing = broker.list_objects_iter(10, None, None, '3:0049:', ':')622 self.assertEquals(len(listing), 2)623 self.assertEquals(624 [row[0] for row in listing],625 ['3:0049:', '3:0049:0049'])626 # Same as above, but using the path argument, so nothing should be627 # returned since path uses a '/' as a delimiter.628 listing = broker.list_objects_iter(10, None, None, None, None,629 '3:0049')630 self.assertEquals(len(listing), 0)631 listing = broker.list_objects_iter(2, None, None, '3:', ':')632 self.assertEquals(len(listing), 2)633 self.assertEquals([row[0] for row in listing], ['3:0000', '3:0000:'])634 listing = broker.list_objects_iter(2, None, None, None, None, '3')635 self.assertEquals(len(listing), 0)636 def test_list_objects_iter_prefix_delim(self):637 # Test ContainerBroker.list_objects_iter638 broker = ContainerBroker(':memory:', account='a', container='c')639 broker.initialize(normalize_timestamp('1'))640 broker.put_object(641 '/pets/dogs/1', normalize_timestamp(0), 0,642 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')643 broker.put_object(644 '/pets/dogs/2', normalize_timestamp(0), 0,645 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')646 broker.put_object(647 '/pets/fish/a', normalize_timestamp(0), 0,648 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')649 broker.put_object(650 '/pets/fish/b', normalize_timestamp(0), 0,651 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')652 broker.put_object(653 '/pets/fish_info.txt', normalize_timestamp(0), 0,654 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')655 broker.put_object(656 '/snakes', normalize_timestamp(0), 0,657 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')658 #def list_objects_iter(self, limit, marker, prefix, delimiter,659 # path=None, format=None):660 listing = broker.list_objects_iter(100, None, None, '/pets/f', '/')661 self.assertEquals([row[0] for row in listing],662 ['/pets/fish/', '/pets/fish_info.txt'])663 listing = broker.list_objects_iter(100, None, None, '/pets/fish', '/')664 self.assertEquals([row[0] for row in listing],665 ['/pets/fish/', '/pets/fish_info.txt'])666 listing = broker.list_objects_iter(100, None, None, '/pets/fish/', '/')667 self.assertEquals([row[0] for row in listing],668 ['/pets/fish/a', '/pets/fish/b'])669 def test_double_check_trailing_delimiter(self):670 # Test ContainerBroker.list_objects_iter for a671 # container that has an odd file with a trailing delimiter672 broker = ContainerBroker(':memory:', account='a', container='c')673 broker.initialize(normalize_timestamp('1'))674 broker.put_object('a', normalize_timestamp(time()), 0,675 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')676 broker.put_object('a/', normalize_timestamp(time()), 0,677 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')678 broker.put_object('a/a', normalize_timestamp(time()), 0,679 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')680 broker.put_object('a/a/a', normalize_timestamp(time()), 0,681 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')682 broker.put_object('a/a/b', normalize_timestamp(time()), 0,683 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')684 broker.put_object('a/b', normalize_timestamp(time()), 0,685 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')686 broker.put_object('b', normalize_timestamp(time()), 0,687 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')688 broker.put_object('b/a', normalize_timestamp(time()), 0,689 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')690 broker.put_object('b/b', normalize_timestamp(time()), 0,691 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')692 broker.put_object('c', normalize_timestamp(time()), 0,693 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')694 broker.put_object('a/0', normalize_timestamp(time()), 0,695 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')696 broker.put_object('0', normalize_timestamp(time()), 0,697 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')698 broker.put_object('0/', normalize_timestamp(time()), 0,699 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')700 broker.put_object('00', normalize_timestamp(time()), 0,701 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')702 broker.put_object('0/0', normalize_timestamp(time()), 0,703 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')704 broker.put_object('0/00', normalize_timestamp(time()), 0,705 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')706 broker.put_object('0/1', normalize_timestamp(time()), 0,707 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')708 broker.put_object('0/1/', normalize_timestamp(time()), 0,709 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')710 broker.put_object('0/1/0', normalize_timestamp(time()), 0,711 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')712 broker.put_object('1', normalize_timestamp(time()), 0,713 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')714 broker.put_object('1/', normalize_timestamp(time()), 0,715 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')716 broker.put_object('1/0', normalize_timestamp(time()), 0,717 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')718 listing = broker.list_objects_iter(25, None, None, None, None)719 self.assertEquals(len(listing), 22)720 self.assertEquals(721 [row[0] for row in listing],722 ['0', '0/', '0/0', '0/00', '0/1', '0/1/', '0/1/0', '00', '1', '1/',723 '1/0', 'a', 'a/', 'a/0', 'a/a', 'a/a/a', 'a/a/b', 'a/b', 'b',724 'b/a', 'b/b', 'c'])725 listing = broker.list_objects_iter(25, None, None, '', '/')726 self.assertEquals(len(listing), 10)727 self.assertEquals(728 [row[0] for row in listing],729 ['0', '0/', '00', '1', '1/', 'a', 'a/', 'b', 'b/', 'c'])730 listing = broker.list_objects_iter(25, None, None, 'a/', '/')731 self.assertEquals(len(listing), 5)732 self.assertEquals(733 [row[0] for row in listing],734 ['a/', 'a/0', 'a/a', 'a/a/', 'a/b'])735 listing = broker.list_objects_iter(25, None, None, '0/', '/')736 self.assertEquals(len(listing), 5)737 self.assertEquals(738 [row[0] for row in listing],739 ['0/', '0/0', '0/00', '0/1', '0/1/'])740 listing = broker.list_objects_iter(25, None, None, '0/1/', '/')741 self.assertEquals(len(listing), 2)742 self.assertEquals(743 [row[0] for row in listing],744 ['0/1/', '0/1/0'])745 listing = broker.list_objects_iter(25, None, None, 'b/', '/')746 self.assertEquals(len(listing), 2)747 self.assertEquals([row[0] for row in listing], ['b/a', 'b/b'])748 def test_double_check_trailing_delimiter_non_slash(self):749 # Test ContainerBroker.list_objects_iter for a750 # container that has an odd file with a trailing delimiter751 broker = ContainerBroker(':memory:', account='a', container='c')752 broker.initialize(normalize_timestamp('1'))753 broker.put_object('a', normalize_timestamp(time()), 0,754 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')755 broker.put_object('a:', normalize_timestamp(time()), 0,756 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')757 broker.put_object('a:a', normalize_timestamp(time()), 0,758 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')759 broker.put_object('a:a:a', normalize_timestamp(time()), 0,760 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')761 broker.put_object('a:a:b', normalize_timestamp(time()), 0,762 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')763 broker.put_object('a:b', normalize_timestamp(time()), 0,764 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')765 broker.put_object('b', normalize_timestamp(time()), 0,766 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')767 broker.put_object('b:a', normalize_timestamp(time()), 0,768 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')769 broker.put_object('b:b', normalize_timestamp(time()), 0,770 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')771 broker.put_object('c', normalize_timestamp(time()), 0,772 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')773 broker.put_object('a:0', normalize_timestamp(time()), 0,774 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')775 broker.put_object('0', normalize_timestamp(time()), 0,776 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')777 broker.put_object('0:', normalize_timestamp(time()), 0,778 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')779 broker.put_object('00', normalize_timestamp(time()), 0,780 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')781 broker.put_object('0:0', normalize_timestamp(time()), 0,782 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')783 broker.put_object('0:00', normalize_timestamp(time()), 0,784 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')785 broker.put_object('0:1', normalize_timestamp(time()), 0,786 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')787 broker.put_object('0:1:', normalize_timestamp(time()), 0,788 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')789 broker.put_object('0:1:0', normalize_timestamp(time()), 0,790 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')791 broker.put_object('1', normalize_timestamp(time()), 0,792 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')793 broker.put_object('1:', normalize_timestamp(time()), 0,794 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')795 broker.put_object('1:0', normalize_timestamp(time()), 0,796 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')797 listing = broker.list_objects_iter(25, None, None, None, None)798 self.assertEquals(len(listing), 22)799 self.assertEquals(800 [row[0] for row in listing],801 ['0', '00', '0:', '0:0', '0:00', '0:1', '0:1:', '0:1:0', '1', '1:',802 '1:0', 'a', 'a:', 'a:0', 'a:a', 'a:a:a', 'a:a:b', 'a:b', 'b',803 'b:a', 'b:b', 'c'])804 listing = broker.list_objects_iter(25, None, None, '', ':')805 self.assertEquals(len(listing), 10)806 self.assertEquals(807 [row[0] for row in listing],808 ['0', '00', '0:', '1', '1:', 'a', 'a:', 'b', 'b:', 'c'])809 listing = broker.list_objects_iter(25, None, None, 'a:', ':')810 self.assertEquals(len(listing), 5)811 self.assertEquals(812 [row[0] for row in listing],813 ['a:', 'a:0', 'a:a', 'a:a:', 'a:b'])814 listing = broker.list_objects_iter(25, None, None, '0:', ':')815 self.assertEquals(len(listing), 5)816 self.assertEquals(817 [row[0] for row in listing],818 ['0:', '0:0', '0:00', '0:1', '0:1:'])819 listing = broker.list_objects_iter(25, None, None, '0:1:', ':')820 self.assertEquals(len(listing), 2)821 self.assertEquals(822 [row[0] for row in listing],823 ['0:1:', '0:1:0'])824 listing = broker.list_objects_iter(25, None, None, 'b:', ':')825 self.assertEquals(len(listing), 2)826 self.assertEquals([row[0] for row in listing], ['b:a', 'b:b'])827 def test_chexor(self):828 broker = ContainerBroker(':memory:', account='a', container='c')829 broker.initialize(normalize_timestamp('1'))830 broker.put_object('a', normalize_timestamp(1), 0,831 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')832 broker.put_object('b', normalize_timestamp(2), 0,833 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')834 hasha = hashlib.md5('%s-%s' % ('a', '0000000001.00000')).digest()835 hashb = hashlib.md5('%s-%s' % ('b', '0000000002.00000')).digest()836 hashc = ''.join(837 ('%2x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))838 self.assertEquals(broker.get_info()['hash'], hashc)839 broker.put_object('b', normalize_timestamp(3), 0,840 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')841 hashb = hashlib.md5('%s-%s' % ('b', '0000000003.00000')).digest()842 hashc = ''.join(843 ('%02x' % (ord(a) ^ ord(b)) for a, b in zip(hasha, hashb)))844 self.assertEquals(broker.get_info()['hash'], hashc)845 def test_newid(self):846 # test DatabaseBroker.newid847 broker = ContainerBroker(':memory:', account='a', container='c')848 broker.initialize(normalize_timestamp('1'))849 id = broker.get_info()['id']850 broker.newid('someid')851 self.assertNotEquals(id, broker.get_info()['id'])852 def test_get_items_since(self):853 # test DatabaseBroker.get_items_since854 broker = ContainerBroker(':memory:', account='a', container='c')855 broker.initialize(normalize_timestamp('1'))856 broker.put_object('a', normalize_timestamp(1), 0,857 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')858 max_row = broker.get_replication_info()['max_row']859 broker.put_object('b', normalize_timestamp(2), 0,860 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')861 items = broker.get_items_since(max_row, 1000)862 self.assertEquals(len(items), 1)863 self.assertEquals(items[0]['name'], 'b')864 def test_sync_merging(self):865 # exercise the DatabaseBroker sync functions a bit866 broker1 = ContainerBroker(':memory:', account='a', container='c')867 broker1.initialize(normalize_timestamp('1'))868 broker2 = ContainerBroker(':memory:', account='a', container='c')869 broker2.initialize(normalize_timestamp('1'))870 self.assertEquals(broker2.get_sync('12345'), -1)871 broker1.merge_syncs([{'sync_point': 3, 'remote_id': '12345'}])872 broker2.merge_syncs(broker1.get_syncs())873 self.assertEquals(broker2.get_sync('12345'), 3)874 def test_merge_items(self):875 broker1 = ContainerBroker(':memory:', account='a', container='c')876 broker1.initialize(normalize_timestamp('1'))877 broker2 = ContainerBroker(':memory:', account='a', container='c')878 broker2.initialize(normalize_timestamp('1'))879 broker1.put_object('a', normalize_timestamp(1), 0,880 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')881 broker1.put_object('b', normalize_timestamp(2), 0,882 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')883 id = broker1.get_info()['id']884 broker2.merge_items(broker1.get_items_since(885 broker2.get_sync(id), 1000), id)886 items = broker2.get_items_since(-1, 1000)887 self.assertEquals(len(items), 2)888 self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))889 broker1.put_object('c', normalize_timestamp(3), 0,890 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')891 broker2.merge_items(broker1.get_items_since(892 broker2.get_sync(id), 1000), id)893 items = broker2.get_items_since(-1, 1000)894 self.assertEquals(len(items), 3)895 self.assertEquals(['a', 'b', 'c'],896 sorted([rec['name'] for rec in items]))897 def test_merge_items_overwrite(self):898 # test DatabaseBroker.merge_items899 broker1 = ContainerBroker(':memory:', account='a', container='c')900 broker1.initialize(normalize_timestamp('1'))901 id = broker1.get_info()['id']902 broker2 = ContainerBroker(':memory:', account='a', container='c')903 broker2.initialize(normalize_timestamp('1'))904 broker1.put_object('a', normalize_timestamp(2), 0,905 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')906 broker1.put_object('b', normalize_timestamp(3), 0,907 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')908 broker2.merge_items(broker1.get_items_since(909 broker2.get_sync(id), 1000), id)910 broker1.put_object('a', normalize_timestamp(4), 0,911 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')912 broker2.merge_items(broker1.get_items_since(913 broker2.get_sync(id), 1000), id)914 items = broker2.get_items_since(-1, 1000)915 self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))916 for rec in items:917 if rec['name'] == 'a':918 self.assertEquals(rec['created_at'], normalize_timestamp(4))919 if rec['name'] == 'b':920 self.assertEquals(rec['created_at'], normalize_timestamp(3))921 def test_merge_items_post_overwrite_out_of_order(self):922 # test DatabaseBroker.merge_items923 broker1 = ContainerBroker(':memory:', account='a', container='c')924 broker1.initialize(normalize_timestamp('1'))925 id = broker1.get_info()['id']926 broker2 = ContainerBroker(':memory:', account='a', container='c')927 broker2.initialize(normalize_timestamp('1'))928 broker1.put_object('a', normalize_timestamp(2), 0,929 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')930 broker1.put_object('b', normalize_timestamp(3), 0,931 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')932 broker2.merge_items(broker1.get_items_since(933 broker2.get_sync(id), 1000), id)934 broker1.put_object('a', normalize_timestamp(4), 0,935 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')936 broker2.merge_items(broker1.get_items_since(937 broker2.get_sync(id), 1000), id)938 items = broker2.get_items_since(-1, 1000)939 self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))940 for rec in items:941 if rec['name'] == 'a':942 self.assertEquals(rec['created_at'], normalize_timestamp(4))943 if rec['name'] == 'b':944 self.assertEquals(rec['created_at'], normalize_timestamp(3))945 self.assertEquals(rec['content_type'], 'text/plain')946 items = broker2.get_items_since(-1, 1000)947 self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))948 for rec in items:949 if rec['name'] == 'a':950 self.assertEquals(rec['created_at'], normalize_timestamp(4))951 if rec['name'] == 'b':952 self.assertEquals(rec['created_at'], normalize_timestamp(3))953 broker1.put_object('b', normalize_timestamp(5), 0,954 'text/plain', 'd41d8cd98f00b204e9800998ecf8427e')955 broker2.merge_items(broker1.get_items_since(956 broker2.get_sync(id), 1000), id)957 items = broker2.get_items_since(-1, 1000)958 self.assertEquals(['a', 'b'], sorted([rec['name'] for rec in items]))959 for rec in items:960 if rec['name'] == 'a':961 self.assertEquals(rec['created_at'], normalize_timestamp(4))962 if rec['name'] == 'b':963 self.assertEquals(rec['created_at'], normalize_timestamp(5))964 self.assertEquals(rec['content_type'], 'text/plain')965def premetadata_create_container_stat_table(self, conn, put_timestamp=None):966 """967 Copied from ContainerBroker before the metadata column was...
CreateParquetN4Folders.py
Source:CreateParquetN4Folders.py
1import boto32s3 = boto3.client('s3')3argo_carrier_visit = s3.put_object(4 Bucket='data-lake-us-west-2-062519970039',5 Body='',6 Key='parquet/n4/argo_carrier_visit/'7 )8argo_chargeable_unit_events = s3.put_object(9 Bucket='data-lake-us-west-2-062519970039',10 Body='',11 Key='parquet/n4/argo_chargeable_unit_events/'12 )13argo_visit_details = s3.put_object(14 Bucket='data-lake-us-west-2-062519970039',15 Body='',16 Key='parquet/n4/argo_visit_details/'17 )18argo_yard = s3.put_object(19 Bucket='data-lake-us-west-2-062519970039',20 Body='',21 Key='parquet/n4/argo_yard/'22 )23crg_bills_of_lading = s3.put_object(24 Bucket='data-lake-us-west-2-062519970039',25 Body='',26 Key='parquet/n4/crg_bills_of_lading/'27 )28crg_bl_goods = s3.put_object(29 Bucket='data-lake-us-west-2-062519970039',30 Body='',31 Key='parquet/n4/crg_bl_goods/'32 )33inv_goods = s3.put_object(34 Bucket='data-lake-us-west-2-062519970039',35 Body='',36 Key='parquet/n4/inv_goods/'37 )38inv_move_event = s3.put_object(39 Bucket='data-lake-us-west-2-062519970039',40 Body='',41 Key='parquet/n4/inv_move_event/'42 )43inv_unit = s3.put_object(44 Bucket='data-lake-us-west-2-062519970039',45 Body='',46 Key='parquet/n4/inv_unit/'47 )48inv_unit_fcy_visit = s3.put_object(49 Bucket='data-lake-us-west-2-062519970039',50 Body='',51 Key='parquet/n4/inv_unit_fcy_visit/'52 )53inv_unit_yrd_visit = s3.put_object(54 Bucket='data-lake-us-west-2-062519970039',55 Body='',56 Key='parquet/n4/inv_unit_yrd_visit/'57 )58inv_wi = s3.put_object(59 Bucket='data-lake-us-west-2-062519970039',60 Body='',61 Key='parquet/n4/inv_wi/'62 )63inv_wi_tracking = s3.put_object(64 Bucket='data-lake-us-west-2-062519970039',65 Body='',66 Key='parquet/n4/inv_wi_tracking/'67 )68inv_wq = s3.put_object(69 Bucket='data-lake-us-west-2-062519970039',70 Body='',71 Key='parquet/n4/inv_wq/'72 )73mns_che_move_statistics = s3.put_object(74 Bucket='data-lake-us-west-2-062519970039',75 Body='',76 Key='parquet/n4/mns_che_move_statistics/'77 )78mns_che_operator_statistics = s3.put_object(79 Bucket='data-lake-us-west-2-062519970039',80 Body='',81 Key='parquet/n4/mns_che_operator_statistics/'82 )83mns_che_session = s3.put_object(84 Bucket='data-lake-us-west-2-062519970039',85 Body='',86 Key='parquet/n4/mns_che_session/'87 )88mns_che_session_period = s3.put_object(89 Bucket='data-lake-us-west-2-062519970039',90 Body='',91 Key='parquet/n4/mns_che_session_period/'92 )93mns_che_status = s3.put_object(94 Bucket='data-lake-us-west-2-062519970039',95 Body='',96 Key='parquet/n4/mns_che_status/'97 )98mns_che_trip_statistics = s3.put_object(99 Bucket='data-lake-us-west-2-062519970039',100 Body='',101 Key='parquet/n4/mns_che_trip_statistics/'102 )103xps_che = s3.put_object(104 Bucket='data-lake-us-west-2-062519970039',105 Body='',106 Key='parquet/n4/xps_che/'107 )108xps_ecevent = s3.put_object(109 Bucket='data-lake-us-west-2-062519970039',110 Body='',111 Key='parquet/n4/xps_ecevent/'112 )113xps_ecuser = s3.put_object(114 Bucket='data-lake-us-west-2-062519970039',115 Body='',116 Key='parquet/n4/xps_ecuser/'117 )118ref_bizunit_scoped= s3.put_object(119 Bucket='data-lake-us-west-2-062519970039',120 Body='',121 Key='parquet/n4/ref_bizunit_scoped/'122 )123ref_carrier_itinerary= s3.put_object(124 Bucket='data-lake-us-west-2-062519970039',125 Body='',126 Key='parquet/n4/ref_carrier_itinerary/'127 )128ref_carrier_service= s3.put_object(129 Bucket='data-lake-us-west-2-062519970039',130 Body='',131 Key='parquet/n4/ref_carrier_service/'132 )133ref_country= s3.put_object(134 Bucket='data-lake-us-west-2-062519970039',135 Body='',136 Key='parquet/n4/ref_country/'137 )138ref_equip_type = s3.put_object(139 Bucket='data-lake-us-west-2-062519970039',140 Body='',141 Key='parquet/n4/ref_equip_type/'142 )143ref_equipment= s3.put_object(144 Bucket='data-lake-us-west-2-062519970039',145 Body='',146 Key='parquet/n4/ref_equipment/'147 )148ref_line_operator = s3.put_object(149 Bucket='data-lake-us-west-2-062519970039',150 Body='',151 Key='parquet/n4/ref_line_operator/'152 )153ref_routing_point = s3.put_object(154 Bucket='data-lake-us-west-2-062519970039',155 Body='',156 Key='parquet/n4/ref_routing_point/'157 )158ref_unloc_code = s3.put_object(159 Bucket='data-lake-us-west-2-062519970039',160 Body='',161 Key='parquet/n4/ref_unloc_code/'162 )163srv_event = s3.put_object(164 Bucket='data-lake-us-west-2-062519970039',165 Body='',166 Key='parquet/n4/srv_event/'167 )168srv_event_types = s3.put_object(169 Bucket='data-lake-us-west-2-062519970039',170 Body='',171 Key='parquet/n4/srv_event_types/'172 )173road_truck_actions = s3.put_object(174 Bucket='data-lake-us-west-2-062519970039',175 Body='',176 Key='parquet/n4/road_truck_actions/'177 )178road_truck_company_drivers = s3.put_object(179 Bucket='data-lake-us-west-2-062519970039',180 Body='',181 Key='parquet/n4/road_truck_company_drivers/'182 )183road_truck_drivers = s3.put_object(184 Bucket='data-lake-us-west-2-062519970039',185 Body='',186 Key='parquet/n4/road_truck_drivers/'187 )188road_truck_transaction_stages = s3.put_object(189 Bucket='data-lake-us-west-2-062519970039',190 Body='',191 Key='parquet/n4/road_truck_transaction_stages/'192 )193road_truck_transactions = s3.put_object(194 Bucket='data-lake-us-west-2-062519970039',195 Body='',196 Key='parquet/n4/road_truck_transactions/'197 )198road_truck_visit_details = s3.put_object(199 Bucket='data-lake-us-west-2-062519970039',200 Body='',201 Key='parquet/n4/road_truck_visit_details/'202 )203road_truck_visit_stages = s3.put_object(204 Bucket='data-lake-us-west-2-062519970039',205 Body='',206 Key='parquet/n4/road_truck_visit_stages/'207 )208road_truck_visit_stats = s3.put_object(209 Bucket='data-lake-us-west-2-062519970039',210 Body='',211 Key='parquet/n4/road_truck_visit_stats/'212 )213vsl_vessel_classes = s3.put_object(214 Bucket='data-lake-us-west-2-062519970039',215 Body='',216 Key='parquet/n4/vsl_vessel_classes/'217 )218vsl_vessel_visit_details = s3.put_object(219 Bucket='data-lake-us-west-2-062519970039',220 Body='',221 Key='parquet/n4/vsl_vessel_visit_details/'222 )223vsl_vessel_visit_lines = s3.put_object(224 Bucket='data-lake-us-west-2-062519970039',225 Body='',226 Key='parquet/n4/vsl_vessel_visit_lines/'227 )228vsl_vessels = s3.put_object(229 Bucket='data-lake-us-west-2-062519970039',230 Body='',231 Key='parquet/n4/vsl_vessels/'...
bucket_automation.py
Source:bucket_automation.py
...42 """43 - Create Public Key Paths44 - Have to supply Body parameter in order to create directory45 """46 s3.put_object(Bucket=bucket_name, Body='', Key='public/')47 # NESDIS key path and its sub directories48 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/')49 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/CSB/')50 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/GOES/')51 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/H8/')52 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/URMA/')53 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/Review/')54 s3.put_object(Bucket=bucket_name, Body='', Key='public/NESDIS/SAB/')55 s3.put_object(Bucket=bucket_name, Body='', Key='public/NMFS/')56 s3.put_object(Bucket=bucket_name, Body='', Key='public/NOS/')57 s3.put_object(Bucket=bucket_name, Body='', Key='public/NWS/')58 s3.put_object(Bucket=bucket_name, Body='', Key='public/OMAO/')59 s3.put_object(Bucket=bucket_name, Body='', Key='public/OAR/')60 # Create Public Key Paths61 s3.put_object(Bucket=bucket_name, Body='', Key='private/')62 # NESDIS key path and its sub directories63 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/')64 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/CSB/')65 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/GOES/')66 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/H8/')67 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/URMA/')68 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/Review/')69 s3.put_object(Bucket=bucket_name, Body='', Key='private/NESDIS/SAB/')70 s3.put_object(Bucket=bucket_name, Body='', Key='private/NMFS/')71 s3.put_object(Bucket=bucket_name, Body='', Key='private/NOS/')72 s3.put_object(Bucket=bucket_name, Body='', Key='private/NWS/')73 s3.put_object(Bucket=bucket_name, Body='', Key='private/OMAO/')74 s3.put_object(Bucket=bucket_name, Body='', Key='private/OAR/')75 # Set CORS bucket config76 cors_config = {77 'CORSRules' : [78 {79 "AllowedHeaders": [80 "*"81 ],82 "AllowedMethods": [83 "PUT",84 "POST",85 "DELETE",86 "GET"87 ],88 "AllowedOrigins": [...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run localstack automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
