How to use input_event method in ATX
Best Python code snippet using ATX
test_transformation_functions.py
Source:test_transformation_functions.py 
1import unittest2import json3from managealooma.transformation_functions import add_column_based_on_null, add_columns_with_default, add_composite_key, \4 add_duplicate_fields, add_suffix, convert_all_event_fields_to_snake_case, convert_dictionary_fields_to_string, \5 convert_empty_value_to_none, convert_event_type_case, convert_null_to_zero, convert_spaces_and_special_characters_to_underscore, \6 convert_string_to_snake_case, convert_values_to_none, flatten_json, flatten_json_1_level, map_key_in_dictionary_to_value, \7 map_value_in_list_to_dictionary_key, mark_for_delete, parse_list_of_json_and_concat, remove_duplicate_field, remove_outer_key, \8 remove_starting_characters_from_keys, remove_whitespace, rename_fields, split_event_to_multiple_events, split_field_list_to_multiple_events, \9 whitelist_or_blacklist_columns10class TestTransformationFunctions(unittest.TestCase):11 # Test 112 def test_add_column_based_on_null_when_null(self):13 input = {'last_4': None}14 output = {'last_4': 'N'}15 self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null=None))16 # Test 217 def test_add_column_based_on_null_when_null_both_params_entered(self):18 input = {'last_4': None}19 output = {'last_4': 'N'}20 self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null='Y'))21 # Test 322 def test_add_column_based_on_null_when_not_null(self):23 input = {'last_4': 0000}24 output = {'last_4': 'Y'}25 self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null=None, new_value_if_not_null='Y'))26 # Test 427 def test_add_column_based_on_null_when_not_null_both_params_entered(self):28 input = {'last_4': 0000}29 output = {'last_4': 'Y'}30 self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null='Y'))31 # Test 532 def test_add_columns_with_default_add_columns(self):33 stuff_to_add = {'add_me': False, 'and_me': 123}34 input_event = {'just_one_key_to_start': 'stuff'}35 expected_output_event = {'add_me': False,36 'and_me': 123,37 'just_one_key_to_start': 'stuff'}38 self.assertEqual(expected_output_event, add_columns_with_default(input_event, field_and_default_dict=stuff_to_add))39 # Test 640 def test_add_columns_with_default_add_columns_skip_existing(self):41 stuff_to_add = {'add_me': False, 'skip_me': 123}42 input_event = {'just_one_key_to_start': 'stuff',43 'skip_me': 'please do not change me'}44 expected_output_event = {'add_me': False,45 'skip_me': 'please do not change me',46 'just_one_key_to_start': 'stuff'}47 self.assertEqual(expected_output_event, add_columns_with_default(input_event, field_and_default_dict=stuff_to_add))48 # Test 749 def test_add_composite_key_key(self):50 input_event = {'product_id': 123456,51 'product_name': 'my_product'}52 expected_output_event = {'product_id': 123456,53 'product_name': 'my_product',54 'product_id_name': '123456-my_product'}55 self.assertEqual(expected_output_event, add_composite_key(input_event, field_list=['product_id', 'product_name'], key_name='product_id_name'))56 # Test 857 def test_add_duplicate_fields_single(self):58 suffix_to_add = 'add'59 input_event = {'start_with_me': 123}60 expected_output_event = {'start_with_me': 123,61 'start_with_me_add': 123}62 self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffix_to_add, keep_original=True))63 # Test 964 def test_add_duplicate_fields_multiple(self):65 suffixes_to_add = ['add', 'add_me_too']66 input_event = {'start_with_me': 123}67 expected_output_event = {'start_with_me': 123,68 'start_with_me_add': 123,69 'start_with_me_add_me_too': 123}70 self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffixes_to_add, keep_original=True))71 # Test 1072 def test_add_duplicate_fields_multiple_remove_original(self):73 suffixes_to_add = ['add', 'add_me_too']74 input_event = {'start_with_me': 123}75 expected_output_event = {'start_with_me_add': 123,76 'start_with_me_add_me_too': 123}77 self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffixes_to_add, keep_original=False))78 # Test 1179 def test_add_suffix_single_field_default_separator(self):80 input_event = {'leave_me': 'my_name_stays',81 'suffix_me': 'I want to be suffix_me_withthis'}82 expected_output_event = {'leave_me': 'my_name_stays',83 'suffix_me_withthis': 'I want to be suffix_me_withthis'}84 self.assertEqual(expected_output_event, add_suffix(input_event, fields='suffix_me', suffix='withthis'))85 # Test 1286 def test_add_suffix_two_fields_default_separator(self):87 input_event = {'leave_me': 'my_name_stays',88 'suffix_me': 'I want to be suffix_me_withthis',89 'suffix_me_too': 'I want to be suffix_me_withthis'}90 expected_output_event = {'leave_me': 'my_name_stays',91 'suffix_me_withthis': 'I want to be suffix_me_withthis',92 'suffix_me_too_withthis': 'I want to be suffix_me_withthis'}93 self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'suffix_me_too'], suffix='withthis'))94 # Test 1395 def test_add_suffix_single_field_new_separator(self):96 input_event = {'leave_me': 'my_name_stays',97 'suffix_me': 'I want to be suffix_me---withthis'}98 expected_output_event = {'leave_me': 'my_name_stays',99 'suffix_me---withthis': 'I want to be suffix_me---withthis'}100 self.assertEqual(expected_output_event, add_suffix(input_event, fields='suffix_me', suffix='withthis', separator='---'))101 # Test 14102 def test_add_suffix_two_fields_new_separator(self):103 input_event = {'leave_me': 'my_name_stays',104 'suffix_me': 'I want to be suffix_me---withthis',105 'suffix_me_too': 'I want to be suffix_me---withthis'}106 expected_output_event = {'leave_me': 'my_name_stays',107 'suffix_me---withthis': 'I want to be suffix_me---withthis',108 'suffix_me_too---withthis': 'I want to be suffix_me---withthis'}109 self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'suffix_me_too'], suffix='withthis', separator='---'))110 # Test 15111 def test_add_suffix_field_not_in_event(self):112 input_event = {'leave_me': 'my_name_stays',113 'suffix_me': 'I want to be suffix_me---withthis'}114 expected_output_event = {'leave_me': 'my_name_stays',115 'suffix_me---withthis': 'I want to be suffix_me---withthis'}116 self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'skip_me'], suffix='withthis', separator='---'))117 # Test 16118 def test_convert_all_event_fields_to_snake_case(self):119 input_event = {'MakeThisSnakeCase': 'stuff_in_field',120 'Make This Snake Case': 'stuff_in_field',121 'keep_this_snake_case': 'stuff_in_field',122 'makeThisSnakeCase': 'stuff_in_field',123 'leavemealone': 'left this one alone',124 '_metadata': {'event_type': 'ObjectName',125 'input_label': 'Label',126 'input_type': 'salesforce'}}127 output_event = {'make_this_snake_case': 'stuff_in_field',128 'make_this_snake_case': 'stuff_in_field',129 'keep_this_snake_case': 'stuff_in_field',130 'make_this_snake_case': 'stuff_in_field',131 'leavemealone': 'left this one alone',132 '_metadata': {'event_type': 'ObjectName',133 'input_label': 'Label',134 'input_type': 'salesforce'}}135 self.assertEqual(output_event, convert_all_event_fields_to_snake_case(input_event))136 # Test 17137 def test_convert_dictionary_fields_to_string_single_dictionary_is_string(self):138 input_event = {'dump me': {'make me': 'a string',139 'change me too': 'into - string'}}140 output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list='dump me')141 self.assertTrue(isinstance(output_event['dump me'], str))142 # Test 18143 def test_convert_dictionary_fields_to_string_single_dictionary_is_not_dict(self):144 input_event = {'dump me': {'make me': 'a string',145 'change me too': 'into - string'}}146 output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list='dump me')147 self.assertFalse(isinstance(output_event['dump me'], dict))148 # Test 19149 def test_convert_dictionary_fields_to_string_multiple_dictionary_is_string(self):150 input_event = {'dump me': {'make me': 'a string',151 'change me too': 'into - string'},152 'dump me too': {'make me': 'a string',153 'change me too': 'into - string'}154 }155 output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me', 'dump me too'])156 self.assertTrue(isinstance(output_event['dump me'], str) and isinstance(output_event['dump me too'], str))157 # Test 20158 def test_convert_dictionary_fields_to_string_multiple_dictionary_only_first_is_string(self):159 input_event = {'dump me': {'make me': 'a string',160 'change me too': 'into - string'},161 'let me be': {'leave me': 'a dict',162 'i do not like': 'to change'}163 }164 output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me'])165 self.assertTrue(isinstance(output_event['dump me'], str))166 # Test 21167 def test_convert_dictionary_fields_to_string_multiple_dictionary_not_specified_is_still_a_dict(self):168 input_event = {'dump me': {'make me': 'a string',169 'change me too': 'into - string'},170 'let me be': {'leave me': 'a dict',171 'i do not like': 'to change'}172 }173 output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me'])174 self.assertTrue(isinstance(output_event['let me be'], dict))175 # Test 22176 def test_convert_empty_value_to_none_list_to_null(self):177 input_event = {'empty_list': [],178 'full_list': ['stuff', 'more_stuff']}179 expected_output_event = {'empty_list': None,180 'full_list': ['stuff', 'more_stuff']}181 self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_list'))182 # Test 23183 def test_convert_empty_value_to_none_list_string_to_null(self):184 input_event = {'empty_list': '[]',185 'full_list': ['stuff', 'more_stuff']}186 expected_output_event = {'empty_list': None,187 'full_list': ['stuff', 'more_stuff']}188 self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_list'))189 # Test 24190 def test_convert_empty_value_to_none_dict_to_null(self):191 input_event = {'empty_dict': {},192 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}193 expected_output_event = {'empty_dict': None,194 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}195 self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_dict'))196 # Test 25197 def test_convert_empty_value_to_none_dict_string_to_null(self):198 input_event = {'empty_dict': '{}',199 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}200 expected_output_event = {'empty_dict': None,201 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}202 self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_dict'))203 # Test 26204 def test_convert_empty_value_to_none_string(self):205 input = {'winner': ''}206 expected_output = {'winner': None}207 self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))208 # Test 27209 def test_convert_empty_value_to_none_not_empty(self):210 input = {'winner': 'not empty'}211 expected_output = {'winner': 'not empty'}212 self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))213 # Test 28214 def test_convert_empty_value_to_none_string_with_spaces_only_to_none(self):215 input = {'winner': ' '}216 expected_output = {'winner': None}217 self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))218 # test 29219 def test_convert_event_type_case_single_event_force_upper(self):220 input_event = {'_metadata': {'event_type': 'my_schema.my_table'}}221 expected_output_event = {'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE'}}222 self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=True))223 # test 30224 def test_convert_event_type_case_list_event_force_upper(self):225 input_event = [{'_metadata': {'event_type': 'my_schema.my_table_one'}},226 {'_metadata': {'event_type': 'My_schema.My_table_Two'}}]227 expected_output_event = [{'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_ONE'}},228 {'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_TWO'}}]229 self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=True))230 # test 31231 def test_convert_event_type_case_single_event_force_lower(self):232 input_event = {'_metadata': {'event_type': 'my_scheMA.my_table'}}233 expected_output_event = {'_metadata': {'event_type': 'my_schema.my_table'}}234 self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=False))235 # test 32236 def test_convert_event_type_case_list_event_force_lower(self):237 input_event = [{'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_ONE'}},238 {'_metadata': {'event_type': 'My_scheMA.MY_table_two'}}]239 expected_output_event = [{'_metadata': {'event_type': 'my_schema.my_table_one'}},240 {'_metadata': {'event_type': 'my_schema.my_table_two'}}]241 self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=False))242 # Test 33243 def test_convert_null_to_zero_list(self):244 input = {'a': None, 'b': None, 'c': 40, 'd': 20}245 expected_output = {'a': None, 'b': 0, 'c': 40, 'd': 20}246 self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list=['b', 'c', 'd']))247 # Test 34248 def test_convert_null_to_zero_single_field(self):249 input = {'a': None, 'b': None, 'c': 40, 'd': 20}250 expected_output = {'a': None, 'b': 0, 'c': 40, 'd': 20}251 self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list='b'))252 # Test 35253 def test_convert_null_to_zero_single_field_non_zero(self):254 input = {'a': None, 'b': None, 'c': 40, 'd': 20}255 expected_output = {'a': None, 'b': None, 'c': 40, 'd': 20}256 self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list='c'))257 # Test 36258 def test_convert_spaces_and_special_characters_to_underscore(self):259 input_string = '$Scr "get-rid^-of-the@" special #characters%&space'260 output_string = '_scr__get_rid__of_the___special__characters__space'261 self.assertEqual(output_string, convert_spaces_and_special_characters_to_underscore(input_string))262 # Test 37263 def test_convert_string_to_snake_case_from_title_case_without_spaces(self):264 input_string = 'MakeThisSnakeCase'265 output_string = 'make_this_snake_case'266 self.assertEqual(output_string, convert_string_to_snake_case(input_string))267 # Test 38268 def test_convert_string_to_snake_case_from_title_case_with_spaces(self):269 input_string = 'Make This Snake Case'270 output_string = 'make_this_snake_case'271 self.assertEqual(output_string, convert_string_to_snake_case(input_string))272 # Test 39273 def test_convert_string_to_snake_case_already_snake_case(self):274 input_string = 'keep_this_snake_case'275 output_string = 'keep_this_snake_case'276 self.assertEqual(output_string, convert_string_to_snake_case(input_string))277 # Test 40278 def test_convert_string_to_snake_case_from_camel_case(self):279 input_string = 'makeThisSnakeCase'280 output_string = 'make_this_snake_case'281 self.assertEqual(output_string, convert_string_to_snake_case(input_string))282 # Test 41283 def test_convert_values_to_none_single_field_all_values(self):284 input_event = {'leave_alone': 1,285 'change_to_none': 'NaN'}286 output_event = {'leave_alone': 1,287 'change_to_none': None}288 self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none'))289 # Test 42290 def test_convert_values_to_none_multiple_fields_all_values(self):291 input_event = {'leave_alone': 1,292 'change_to_none': 'NaN',293 'change_me_too': 5.75}294 output_event = {'leave_alone': 1,295 'change_to_none': None,296 'change_me_too': None}297 self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list=['change_to_none', 'change_me_too']))298 # Test 43299 def test_convert_values_to_none_single_field_single_value_change(self):300 input_event = {'leave_alone': 1,301 'change_to_none': 'NaN'}302 output_event = {'leave_alone': 1,303 'change_to_none': None}304 self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none', field_values='NaN'))305 # Test 44306 def test_convert_values_to_none_single_field_single_value_dont_change(self):307 input_event = {'leave_alone': 1,308 'change_to_none': 'Something Else'}309 output_event = {'leave_alone': 1,310 'change_to_none': 'Something Else'}311 self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none', field_values='NaN'))312 # Test 45313 def test_convert_values_to_none_single_field_value_list_change(self):314 input_event = {'leave_alone': 1,315 'change_to_none': 'NaN',316 'also_change_to_none': 2}317 output_event = {'leave_alone': 1,318 'change_to_none': None,319 'also_change_to_none': None}320 self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list=['change_to_none', 'also_change_to_none'], field_values=['NaN', 2]))321 # test 46322 def test_flatten_json_input_is_list_keep_original_true(self):323 input = {"batters": ['chocolate', 'strawberry', 'vanilla']}324 expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}325 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True)326 self.assertEqual(expected_output, output_event)327 # test 47328 def test_flatten_json_input_is_list_keep_original_false(self):329 input = {"batters": ['chocolate', 'strawberry', 'vanilla']}330 expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}331 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False)332 self.assertEqual(expected_output, output_event)333 # test 48334 def test_flatten_json_input_is_empty_dict_keep_original_false(self):335 input = {"batters": {}}336 expected_output = {}337 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False)338 self.assertEqual(expected_output, output_event)339 # test 49340 def test_flatten_json_input_is_empty_dict_keep_original_true(self):341 input = {"batters": {}}342 expected_output = {"batters": {}}343 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True)344 self.assertEqual(expected_output, output_event)345 # Test 50346 def test_flatten_json_from_string_1_level_keep_original_false_new_keep_as_json(self):347 input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}348 expected_output = {'cones_99 flake': '2.4',349 'cones_cake': {},350 'cones_sugar': [],351 'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}}352 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=False)353 self.assertEqual(expected_output, output_event)354 # Test 51355 def test_flatten_json_from_string_1_level_keep_original_true_new_keep_as_json(self):356 input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}357 expected_output = {'cones': {'99 Flake': '2.4',358 'cake': {},359 'sugar': [],360 'waffle': {'chocolate': 'No',361 'vanilla': {'extra_charge': '2', 'sprinkles': 'Yes'}}},362 'cones_99 flake': '2.4',363 'cones_cake': {},364 'cones_sugar': [],365 'cones_waffle': {'chocolate': 'No',366 'vanilla': {'extra_charge': '2', 'sprinkles': 'Yes'}}}367 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True)368 self.assertEqual(expected_output, output_event)369 # Test 52370 def test_flatten_json_from_string_1_level_keep_original_false_dump_to_string(self):371 input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}372 expected_output = {'cones_99 flake': '2.4',373 'cones_cake': '{}',374 'cones_sugar': [],375 'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}'}376 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=False, dump_to_string=True)377 self.assertEqual(expected_output, output_event)378 # Test 53379 def test_flatten_json_from_string_1_level_keep_original_true_dump_to_string(self):380 input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}381 expected_output = {'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',382 'cones_99 flake': '2.4',383 'cones_cake': '{}',384 'cones_sugar': [],385 'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}'}386 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=True)387 self.assertEqual(expected_output, output_event)388 # Test 54389 def test_flatten_json_from_dict_1_level_keep_original_false_dump_to_string(self):390 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},391 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},392 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}393 expected_output = {'batters_blueberry': '{"creaminess": "light", "sweetness": "super"}',394 'batters_chocolate': '{"creaminess": "moderate", "sweetness": "very"}',395 'batters_vanilla': '{"creaminess": "very", "sweetness": "medium"}'}396 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False, dump_to_string=True)397 self.assertEqual(expected_output, output_event)398 # Test 55399 def test_flatten_json_from_dict_1_level_keep_original_false_keep_as_json(self):400 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},401 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},402 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}403 expected_output = {'batters_blueberry': {"creaminess": "light", "sweetness": "super"},404 'batters_chocolate': {"creaminess": "moderate", "sweetness": "very"},405 'batters_vanilla': {"creaminess": "very", "sweetness": "medium"}}406 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False, dump_to_string=False)407 self.assertEqual(expected_output, output_event)408 # Test 56409 def test_flatten_json_from_dict_1_level_keep_original_true_dump_to_string(self):410 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},411 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},412 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}413 expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '414 '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '415 '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',416 'batters_blueberry': '{"creaminess": "light", "sweetness": "super"}',417 'batters_chocolate': '{"creaminess": "moderate", "sweetness": "very"}',418 'batters_vanilla': '{"creaminess": "very", "sweetness": "medium"}'}419 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=True)420 self.assertEqual(expected_output, output_event)421 # Test 57422 def test_flatten_json_from_dict_1_level_keep_original_true_keep_as_json(self):423 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},424 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},425 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}426 expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},427 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},428 "Blueberry": {"creaminess": "light", "sweetness": "super"}},429 'batters_blueberry': {"creaminess": "light", "sweetness": "super"},430 'batters_chocolate': {"creaminess": "moderate", "sweetness": "very"},431 'batters_vanilla': {"creaminess": "very", "sweetness": "medium"}}432 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)433 self.assertEqual(expected_output, output_event)434 # Test 58435 def test_flatten_json_from_dict_2_level_keep_original_false(self):436 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},437 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},438 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}439 expected_output = {'batters_blueberry_creaminess': 'light',440 'batters_blueberry_sweetness': 'super',441 'batters_chocolate_creaminess': 'moderate',442 'batters_chocolate_sweetness': 'very',443 'batters_vanilla_creaminess': 'very',444 'batters_vanilla_sweetness': 'medium'}445 output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=False)446 self.assertEqual(expected_output, output_event)447 # Test 59448 def test_flatten_json_from_dict_2_level_keep_original_true_as_string(self):449 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},450 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},451 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}452 expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '453 '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '454 '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',455 'batters_blueberry_creaminess': 'light',456 'batters_blueberry_sweetness': 'super',457 'batters_chocolate_creaminess': 'moderate',458 'batters_chocolate_sweetness': 'very',459 'batters_vanilla_creaminess': 'very',460 'batters_vanilla_sweetness': 'medium'}461 output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=True, dump_to_string=True)462 self.assertEqual(expected_output, output_event)463 # Test 60464 def test_flatten_json_from_dict_2_level_keep_original_true_keep_as_json(self):465 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},466 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},467 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}468 expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},469 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},470 "Blueberry": {"creaminess": "light", "sweetness": "super"}},471 'batters_blueberry_creaminess': 'light',472 'batters_blueberry_sweetness': 'super',473 'batters_chocolate_creaminess': 'moderate',474 'batters_chocolate_sweetness': 'very',475 'batters_vanilla_creaminess': 'very',476 'batters_vanilla_sweetness': 'medium'}477 output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=True, dump_to_string=False)478 self.assertEqual(expected_output, output_event)479 # Test 61480 def test_flatten_json_list_n_levels_2_fields(self):481 input = {482 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",483 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"484 }485 expected_output = {486 'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',487 'cones_99 flake': '2.4',488 'cones_cake': '{}',489 'cones_sugar': [],490 'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}',491 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",492 "ice_cream_chocolate": 1.99,493 "ice_cream_vanilla": 0.99}494 output_event = flatten_json(input, field_or_field_list=['cones', 'ice_cream'], levels=1,495 keep_original=True, dump_to_string=True)496 self.assertEqual(expected_output, output_event)497 # Test 62498 def test_flatten_json_from_string_3_level_keep_original_false_new(self):499 input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}500 expected_output = {'cones_99 flake': 2.4,501 'cones_sugar': [],502 'cones_waffle_chocolate': 'No',503 'cones_waffle_vanilla_extra_charge': '2',504 'cones_waffle_vanilla_sprinkles': 'Yes'}505 output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=False)506 self.assertEqual(expected_output, output_event)507 # Test 63508 def test_flatten_json_from_string_3_level_keep_original_true_new(self):509 input = {510 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}511 expected_output = {512 'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',513 'cones_99 flake': 2.4,514 'cones_sugar': [],515 'cones_waffle_chocolate': 'No',516 'cones_waffle_vanilla_extra_charge': '2',517 'cones_waffle_vanilla_sprinkles': 'Yes'518 }519 output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=True, dump_to_string=True)520 self.assertEqual(expected_output, output_event)521 # Test 64522 def test_flatten_json_capital_field(self):523 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},524 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},525 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}526 expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '527 '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '528 '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',529 'batters_blueberry_creaminess': 'light',530 'batters_blueberry_sweetness': 'super',531 'batters_chocolate_creaminess': 'moderate',532 'batters_chocolate_sweetness': 'very',533 'batters_vanilla_creaminess': 'very',534 'batters_vanilla_sweetness': 'medium'}535 output_event = flatten_json(input, field_or_field_list='Batters', levels=2, keep_original=True, dump_to_string=True)536 self.assertEqual(expected_output, output_event)537 # Test 65538 def test_flatten_json_list_n_levels_1_field(self):539 input = {540 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",541 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"542 }543 expected_output = {544 'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',545 'cones_99 flake': '2.4',546 'cones_cake': '{}',547 'cones_sugar': [],548 'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}',549 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",550 }551 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=True)552 self.assertEqual(expected_output, output_event)553 # Test 66554 def test_flatten_json_from_string_3_level_keep_original_true_dump_string_false(self):555 input = {556 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}557 expected_output = {558 'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},559 'cones_99 flake': 2.4,560 'cones_sugar': [],561 'cones_waffle_chocolate': 'No',562 'cones_waffle_vanilla_extra_charge': '2',563 'cones_waffle_vanilla_sprinkles': 'Yes'564 }565 output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=True, dump_to_string=False)566 self.assertEqual(expected_output, output_event)567 # Test 67568 def test_flatten_json_input_is_list_keep_original_true_dump_string_false(self):569 input = {"batters": ['chocolate', 'strawberry', 'vanilla']}570 expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}571 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)572 self.assertEqual(expected_output, output_event)573 # Test 68574 def test_flatten_json_input_is_empty_dict_keep_original_true_dump_string_false(self):575 input = {"batters": {}}576 expected_output = {"batters": {}}577 output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)578 self.assertEqual(expected_output, output_event)579 # Test 69580 def test_flatten_json_capital_field_dump_string_false(self):581 input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},582 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},583 "Blueberry": {"creaminess": "light", "sweetness": "super"}}}584 expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},585 "Chocolate": {"creaminess": "moderate", "sweetness": "very"},586 "Blueberry": {"creaminess": "light", "sweetness": "super"}},587 'batters_blueberry_creaminess': 'light',588 'batters_blueberry_sweetness': 'super',589 'batters_chocolate_creaminess': 'moderate',590 'batters_chocolate_sweetness': 'very',591 'batters_vanilla_creaminess': 'very',592 'batters_vanilla_sweetness': 'medium'}593 output_event = flatten_json(input, field_or_field_list='Batters', levels=2, keep_original=True, dump_to_string=False)594 self.assertEqual(expected_output, output_event)595 # Test 70596 def test_flatten_json_list_n_levels_1_field_dump_string_false(self):597 input = {598 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",599 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"600 }601 expected_output = {602 'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},603 'cones_99 flake': '2.4',604 'cones_cake': {},605 'cones_sugar': [],606 'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"},607 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",608 }609 output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=False)610 self.assertEqual(expected_output, output_event)611 # Test 71612 def test_flatten_json_list_n_levels_2_fields_dump_string_false(self):613 input = {614 "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",615 "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"616 }617 expected_output = {618 'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},619 'cones_99 flake': '2.4',620 'cones_cake': {},621 'cones_sugar': [],622 'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"},623 "ice_cream": {"chocolate": 1.99, "vanilla": 0.99},624 "ice_cream_chocolate": 1.99,625 "ice_cream_vanilla": 0.99}626 output_event = flatten_json(input, field_or_field_list=['cones', 'ice_cream'], levels=1,627 keep_original=True, dump_to_string=False)628 self.assertEqual(expected_output, output_event)629 # Test 72630 def test_flatten_json_1_level_and_dumps_using_ast(self):631 input_event = {'my_field': "{\"a\": None, \"b\": 11, \"c\": 3297, \"d\": 1497, \"e\": 11}"}632 output_event = {'my_field_a': None,633 'my_field_b': 11,634 'my_field_c': 3297,635 'my_field_d': 1497,636 'my_field_e': 11}637 self.assertEqual(output_event, flatten_json_1_level(event=input_event, field_name='my_field', field_name_underscore='my_field_', dump_to_string=True))638 # Test 73639 def test_map_key_in_dictionary_to_value_clean_transform(self):640 mapping_dict = {1: 'first',641 2: 'second',642 3: 'third'}643 input_event = {'number_field': 2,644 '_metadata': {'event_type': 'event_type_name'}645 }646 expected_output_event = {'number_field': 2,647 'number_field_with_name': 'second',648 '_metadata': {'event_type': 'event_type_name'}}649 self.assertEqual(expected_output_event, map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False))650 # Test 74651 def test_map_key_in_dictionary_to_value_null_allowed(self):652 mapping_dict = {1: 'first',653 2: 'second',654 3: 'third'}655 input_event = {'number_field': None,656 '_metadata': {'event_type': 'event_type_name'}657 }658 expected_output_event = {'number_field': None,659 'number_field_with_name': None,660 '_metadata': {'event_type': 'event_type_name'}}661 self.assertEqual(expected_output_event, map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=True))662 # Test 75663 def test_map_key_in_dictionary_to_value_missing_error(self):664 mapping_dict = {1: 'first',665 2: 'second',666 3: 'third'}667 input_event = {'number_field': 7,668 '_metadata': {'event_type': 'event_type_name'}669 }670 with self.assertRaises(BaseException) as cm:671 map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False)672 self.assertEqual(str(cm.exception), 'Missing enum transform event_type_name number_field')673 # Test 76674 def test_map_key_in_dictionary_to_value_value_null_error(self):675 mapping_dict = {1: 'first',676 2: 'second',677 3: 'third'}678 input_event = {'number_field': None,679 '_metadata': {'event_type': 'event_type_name'}680 }681 with self.assertRaises(BaseException) as cm:682 map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False)683 self.assertEqual(str(cm.exception), 'Missing enum transform event_type_name number_field')684 # Test 77685 def test_map_value_in_list_to_dictionary_key_successful_mapping(self):686 input_event = {"_metadata": {"event_type": "sweets"},687 "type": "donut"}688 breakfast_item_categories = {"baked good": ["donut", "cake", "croissant"],689 "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}690 expected_output_event = {'_metadata': {'event_type': 'sweets'},691 'breakfast_item_category': 'baked good',692 'type': 'donut'}693 self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,694 existing_column='type', new_column='breakfast_item_category',695 allow_nulls=False, passthrough=True))696 # Test 78697 def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_false_passthrough_true(self):698 input_event = {"_metadata": {"event_type": "sweets"},699 "type": "donut"}700 breakfast_item_categories = {"baked good": ["cake", "croissant"],701 "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}702 expected_output_event = {'_metadata': {'event_type': 'sweets'},703 'breakfast_item_category': 'donut',704 'type': 'donut'}705 self.assertEqual(expected_output_event,706 map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,707 existing_column='type', new_column='breakfast_item_category',708 allow_nulls=False, passthrough=True))709 # Test 79710 def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_false_passthrough_false(self):711 input_event = {"_metadata": {"event_type": "sweets"},712 "type": "donut",713 }714 breakfast_item_categories = {"baked_goods": ["cake", "croissant"],715 "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}716 with self.assertRaises(Exception):717 map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,718 existing_column='type',719 new_column='breakfast_item_category',720 allow_nulls=False, passthrough=False)721 # Test 80722 def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_true_passthrough_true(self):723 input_event = {"_metadata": {"event_type": "sweets"},724 "type": "donut"}725 breakfast_item_categories = {"baked good": ["cake", "croissant"],726 "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}727 expected_output_event = {'_metadata': {'event_type': 'sweets'},728 'breakfast_item_category': 'donut',729 'type': 'donut'}730 self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,731 existing_column='type', new_column='breakfast_item_category',732 allow_nulls=True, passthrough=True))733 # Test 81734 def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_true_passthrough_false(self):735 input_event = {"_metadata": {"event_type": "sweets"},736 "type": "donut"}737 breakfast_item_categories = {"baked good": ["cake", "croissant"],738 "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}739 expected_output_event = {'_metadata': {'event_type': 'sweets'},740 'breakfast_item_category': None,741 'type': 'donut'}742 self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,743 existing_column='type', new_column='breakfast_item_category',744 allow_nulls=True, passthrough=False))745 # Test 82746 def test_mark_for_delete(self):747 test_mark_for_delete_input = """{"id": 13,748 "table_name": "alooma_test",749 "primary_key": "456789",750 "old_row_json": {"id":6,751 "name":"User 6",752 "created_at":"2017-08-23T05:01:51.753963",753 "updated_at":"2017-08-26T15:27:13.455902"},754 "_metadata": {"event_type": "test"}}"""755 expected_output_event = {'_metadata': {'event_type': 'alooma_test',756 'table': 'alooma_test'},757 'created_at': '2017-08-23T05:01:51.753963',758 'id': 6,759 'mark_for_delete': True,760 'name': 'User 6',761 'updated_at': '2017-08-26T15:27:13.455902'}762 input_event = json.loads(test_mark_for_delete_input)763 input_event["old_row_json"] = json.dumps(input_event["old_row_json"])764 self.assertEqual(expected_output_event, mark_for_delete(input_event))765 # Test 83766 def test_parse_list_of_json_and_concat_keep_original_true(self):767 input_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},768 {'key_to_concat': 456, 'key_to_ignore': 'def'},769 {'key_to_concat': 789, 'key_to_ignore': 'ghi'}]}770 expected_output_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},771 {'key_to_concat': 456, 'key_to_ignore': 'def'},772 {'key_to_concat': 789, 'key_to_ignore': 'ghi'}],773 'list_of_dicts_key_to_concats': [123, 456, 789]}774 self.assertEqual(expected_output_event, parse_list_of_json_and_concat(input_event, field_name='list_of_dicts', keep_original=True, field_to_keep='key_to_concat'))775 # Test 84776 def test_parse_list_of_json_and_concat_keep_original_false(self):777 input_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},778 {'key_to_concat': 456, 'key_to_ignore': 'def'},779 {'key_to_concat': 789, 'key_to_ignore': 'ghi'}]}780 expected_output_event = {'list_of_dicts_key_to_concats': [123, 456, 789]}781 self.assertEqual(expected_output_event, parse_list_of_json_and_concat(input_event, field_name='list_of_dicts', keep_original=False, field_to_keep='key_to_concat'))782 # Test 85783 def test_remove_duplicate_field_remove(self):784 input_event = {'Im Key': 'I am repeated information',785 'im_key': 'I am repeated information'}786 expected_output_event = {'Im Key': 'I am repeated information'}787 self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='Im Key', field_to_discard='im_key'))788 # Test 86789 def test_remove_duplicate_field_remove_keep_field_none(self):790 input_event = {'Im Key': None,791 'im_key': 'I should be kept'}792 expected_output_event = {'im_key': 'I should be kept'}793 self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='Im Key', field_to_discard='im_key'))794 # Test 87795 def test_remove_duplicate_field_missing_a_field_no_discard(self):796 input_event = {'im useless': None,797 'keep_me': 'I should be kept'}798 expected_output_event = {'im useless': None,799 'keep_me': 'I should be kept'}800 self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='keep_me', field_to_discard='discard_me'))801 # Test 88802 def test_remove_duplicate_field_missing_a_field_keep_discard(self):803 input_event = {'im useless': None,804 'discard_me': 'I stay Anyway'}805 expected_output_event = {'im useless': None,806 'discard_me': 'I stay Anyway'}807 self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='keep_me', field_to_discard='discard_me'))808 # Test 89809 def test_remove_outer_key_remove(self):810 input_event = {'a key': 'stuff',811 'nested_stuff': {'field_one': 1,812 'field_two': 2,813 'field_three': 3}814 }815 expected_output_event = {'a key': 'stuff',816 'field_one': 1,817 'field_two': 2,818 'field_three': 3819 }820 self.assertEqual(expected_output_event, remove_outer_key(input_event, key_name='nested_stuff'))821 # Test 90822 def test_remove_outer_key_no_dict(self):823 input_event = {'a key': 'stuff',824 'nested_stuff': {'field_one': 1,825 'field_two': 2,826 'field_three': 3}827 }828 expected_output_event = {'a key': 'stuff',829 'nested_stuff': {'field_one': 1,830 'field_two': 2,831 'field_three': 3}832 }833 self.assertEqual(expected_output_event, remove_outer_key(input_event, key_name='a key'))834 # Test 91835 def test_remove_outer_key_has_key(self):836 input = {'remove_me': {'i want to be top level': 'keep me',837 'i want to be top level too': 'keep me too'}}838 expected_output = {'i want to be top level': 'keep me',839 'i want to be top level too': 'keep me too'}840 self.assertEqual(expected_output, remove_outer_key(input, key_name='remove_me'))841 # Test 92842 def test_remove_outer_key_missing_key(self):843 input = {'remove_me': {'i want to be top level': 'keep me',844 'i want to be top level too': 'keep me too'}}845 expected_output = {'remove_me': {'i want to be top level': 'keep me',846 'i want to be top level too': 'keep me too'}}847 self.assertEqual(expected_output, remove_outer_key(input, key_name='i_dont_exist'))848 # Test 93849 def test_remove_outer_key_has_key_and_nest(self):850 input = {'remove_me': {'i want to be top level': 'keep me',851 'i want to be top level too': 'keep me too',852 'My children stay nested': {'one down': 'mr one', 'two down': 'ms two', 'has more nest': {'way down': 'mr way', 'way down again': 'ms way'}}}}853 expected_output = {'i want to be top level': 'keep me',854 'i want to be top level too': 'keep me too',855 'My children stay nested': {'one down': 'mr one', 'two down': 'ms two', 'has more nest': {'way down': 'mr way', 'way down again': 'ms way'}}}856 self.assertEqual(expected_output, remove_outer_key(input, key_name='remove_me'))857 # Test 94858 def test_remove_starting_characters_from_keys_one_char(self):859 input_event = {'_metadata': {},860 '$strip_me': 'no more $ in key',861 '$strip_me_too': 'no more $ in key',862 '_dont_strip': 'leave my underscore',863 'nothing_here': 'nothing special'}864 expected_output_event = {'_metadata': {},865 '_dont_strip': 'leave my underscore',866 'nothing_here': 'nothing special',867 'strip_me_too': 'no more $ in key',868 'strip_me': 'no more $ in key'}869 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$'))870 # Test 95871 def test_remove_starting_characters_from_keys_many_char(self):872 input_event = {'_metadata': {},873 'MY$-strip_me': 'no more MY$- in key',874 'MY$-strip_me_too': 'no more MY$- in key',875 '_dont_strip': 'leave_my_underscore',876 'nothing_here': 'nothing_special'}877 expected_output_event = {'_metadata': {},878 '_dont_strip': 'leave_my_underscore',879 'nothing_here': 'nothing_special',880 'strip_me': 'no more MY$- in key',881 'strip_me_too': 'no more MY$- in key'}882 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='MY$-'))883 # Test 96884 def test_remove_starting_characters_from_keys_no_keys_present(self):885 input_event = {'_metadata': {},886 'nothing_here': 'nothing to strip',887 'nothing_here_either': 'nothing to strip'}888 expected_output_event = {'_metadata': {},889 'nothing_here': 'nothing to strip',890 'nothing_here_either': 'nothing to strip'}891 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$'))892 # Test 97893 def test_remove_starting_characters_from_keys_starts_with_part(self):894 input_event = {'_metadata': {},895 '$$strip_me': 'starts with two - strimp em',896 '$strip_me': 'starts with just one - stays here'}897 expected_output_event = {'_metadata': {},898 'strip_me': 'starts with two - strimp em',899 '$strip_me': 'starts with just one - stays here'}900 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$$'))901 # Test 98902 def test_remove_starting_characters_from_sub_field(self):903 input_event = {'$strip_my_subs': {'$strip_me': 'strimp em',904 '$and_me': 'strip em too',905 'not_me': 'I never change'}}906 expected_output_event = {'$strip_my_subs': {'strip_me': 'strimp em',907 'and_me': 'strip em too',908 'not_me': 'I never change'}}909 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$', field_with_json='$strip_my_subs'))910 # Test 99911 def test_remove_starting_characters_from_sub_field_missing_key(self):912 input_event = {'$strip_my_subs': {'$strip_me': 'strimp em',913 '$and_me': 'strip em too',914 'not_me': 'I never change'}}915 expected_output_event = {'$strip_my_subs': {'$strip_me': 'strimp em',916 '$and_me': 'strip em too',917 'not_me': 'I never change'}}918 self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$', field_with_json='not_in_event'))919 # Test 100920 def test_trim_whitespace(self):921 input = {'strip': ' this is my example '}922 output = {'strip': 'this is my example'}923 self.assertEqual(output, remove_whitespace(input, field_or_field_list='strip'))924 # Test 101925 def test_remove_whitespace_list(self):926 input = {'strip': ' this is my example ',927 'strip me too': ' this is my example'}928 output = {'strip': 'this is my example',929 'strip me too': 'this is my example'}930 self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip', 'strip me too']))931 # Test 102932 def test_remove_whitespace_only_specifiy_one(self):933 input = {'strip': ' this is my example ',934 'strip me too': ' this is my example'}935 output = {'strip': 'this is my example',936 'strip me too': ' this is my example'}937 self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip']))938 # Test 103939 def test_remove_whitespace_has_an_int(self):940 input = {'strip': 1}941 output = {'strip': 1}942 self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip']))943 # Test 104944 def test_rename_fields_one_field_in_event(self):945 input_event = {'leave_me': 'my_name_stays',946 'change_me': 'My new name is: i_changed'}947 expected_output_event = {'leave_me': 'my_name_stays',948 'i_changed': 'My new name is: i_changed'}949 self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed'}))950 # Test 105951 def test_rename_fields_one_field_not_in_event(self):952 input_event = {'leave_me': 'my_name_stays',953 'change_me': 'My new name is: i_changed'}954 expected_output_event = {'leave_me': 'my_name_stays',955 'change_me': 'My new name is: i_changed'}956 self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'skip_me': 'not_in_event'}))957 # Test 106958 def test_rename_fields_two_fields_in_event(self):959 input_event = {'leave_me': 'my_name_stays',960 'change_me': 'My new name is: i_changed',961 'change_me_too': 'My new name is: i_changed_too'}962 expected_output_event = {'leave_me': 'my_name_stays',963 'i_changed': 'My new name is: i_changed',964 'i_changed_too': 'My new name is: i_changed_too'}965 self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed', 'change_me_too': 'i_changed_too'}))966 # Test 107967 def test_rename_fields_two_fields_one_in_event(self):968 input_event = {'leave_me': 'my_name_stays',969 'change_me': 'My new name is: i_changed'}970 expected_output_event = {'leave_me': 'my_name_stays',971 'i_changed': 'My new name is: i_changed'}972 self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed', 'skip_me': 'not_in_event'}))973 # Test 108974 def test_rename_fields_raise_exception(self):975 rename_field_dict = {'im_similar': 'properties_user_id',976 'imsimilar': 'properties_user_id'}977 input_event = {'im_similar': 2,978 'imsimilar': 3}979 with self.assertRaises(BaseException):980 rename_fields(input_event, rename_field_dict)981 # Test 109982 def test_split_event_to_multiple_events_skip_for_parent_id(self):983 input_event = {'_metadata': {'event_type': 'schema_name.event_name',984 '@parent_uuid': '123'},985 'a_non_metadata_field': 'some value'}986 expected_output_event = {'_metadata': {'event_type': 'schema_name.event_name',987 '@parent_uuid': '123'},988 'a_non_metadata_field': 'some value'}989 self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['one', 'two']))990 # Test 110991 def test_split_event_to_multiple_events_split_into_two(self):992 input_event = {'_metadata': {'event_type': 'schema_name.event_name',993 '@parent_uuid': ''},994 'a_non_metadata_field': 'some value'}995 expected_output_event = [{'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_one'},996 'a_non_metadata_field': 'some value'},997 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_two'},998 'a_non_metadata_field': 'some value'}]999 self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two']))1000 # Test 1111001 def test_split_event_to_multiple_events_split_into_three(self):1002 input_event = {'_metadata': {'event_type': 'schema_name.event_name',1003 '@parent_uuid': ''},1004 'a_non_metadata_field': 'some value'}1005 expected_output_event = [{'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_one'},1006 'a_non_metadata_field': 'some value'},1007 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_two'},1008 'a_non_metadata_field': 'some value'},1009 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_three'},1010 'a_non_metadata_field': 'some value'}]1011 self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two', 'table_three']))1012 # Test 1121013 def test_split_event_to_multiple_events_throw_exception_if_not_fully_qualified(self):1014 input_event = {'_metadata': {'event_type': 'schema_name_event_name',1015 '@parent_uuid': ''},1016 'a_non_metadata_field': 'some value'}1017 with self.assertRaises(BaseException) as cm:1018 split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two', 'table_three'])1019 self.assertEqual(str(cm.exception), 'Only fully qualified events can be split. Event type must be schema_name.table_name')1020 # Test 1131021 def test_split_field_list_to_multiple_events_split(self):1022 input = {'id': 1,1023 'names': ['first', 'second'],1024 '_metadata': {'uuid': '1a'}}1025 output = [{'id': 1,1026 'name': 'first',1027 'counter': 1,1028 '_metadata': {'uuid': '1a'}},1029 {'id': 1,1030 'name': 'second',1031 'counter': 2,1032 '_metadata': {'uuid': '1a'}}1033 ]1034 self.assertEqual(output, split_field_list_to_multiple_events(event=input, fields_to_split=['names'], add_counter=True, counter_name='counter', reverse=False))1035 # Test 1141036 def test_split_field_list_to_multiple_events_skip_for_parentd_uuid(self):1037 input = {'id': 1,1038 'names': ['first', 'second'],1039 '_metadata': {'@parent_uuid': '1a'}}1040 output = {'id': 1,1041 'names': ['first', 'second'],1042 '_metadata': {'@parent_uuid': '1a'}}1043 self.assertEqual(output, split_field_list_to_multiple_events(event=input, fields_to_split=['names'], add_counter=True, counter_name='counter', reverse=False))1044 # Test 1151045 def test_whitelist_or_blacklist_columns_whitelist(self):1046 input = {"Cookies": None,1047 "cake": None,1048 "brownies": "yum",1049 "iceCream": "mmmm",1050 "_metadata": {"table": "sweets"}}1051 expected_output = {"Cookies": None,1052 "cake": None,1053 "iceCream": "mmmm",1054 "_metadata": {"table": "sweets"}}1055 self.assertEqual(expected_output, whitelist_or_blacklist_columns(input, field_list=['cookies', 'Cake', 'IceCream'], white_or_black_list='whitelist'))1056 # Test 1161057 def test_whitelist_or_blacklist_columns_blacklist(self):1058 input = {"Cookies": None,1059 "cake": None,1060 "brownies": "yum",1061 "iceCream": "mmmm",1062 "_metadata": {"table": "sweets"}}1063 expected_output = {"brownies": "yum",1064 "_metadata": {"table": "sweets"}}...
metrics.py
Source:metrics.py
1# -*- coding: utf-8 -*-2# Copyright 2019 The Chromium OS Authors. All rights reserved.3# Use of this source code is governed by a BSD-style license that can be4# found in the LICENSE file.5"""Metrics for general consumption.6See infra/proto/metrics.proto for a description of the type of record that this7module will be creating.8"""9from __future__ import print_function10import sys11from chromite.lib import cros_logging as logging12from chromite.utils import metrics13assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'14def deserialize_metrics_log(output_events, prefix=None):15 """Read the current metrics events, adding to output_events.16 This layer facilitates converting between the internal17 chromite.utils.metrics representation of metric events and the18 infra/proto/src/chromiumos/metrics.proto output type.19 Args:20 output_events: A chromiumos.MetricEvent protobuf message.21 prefix: A string to prepend to all metric event names.22 """23 timers = {}24 def make_name(name):25 """Prepend a closed-over prefix to the given name."""26 if prefix:27 return '%s.%s' % (prefix, name)28 else:29 return name30 # Reduce over the input events to append output_events.31 for input_event in metrics.read_metrics_events():32 if input_event.op == metrics.OP_START_TIMER:33 timers[input_event.arg] = (input_event.name,34 input_event.timestamp_epoch_millis)35 elif input_event.op == metrics.OP_STOP_TIMER:36 # TODO(wbbradley): Drop the None fallback https://crbug.com/1001909.37 timer = timers.pop(input_event.arg, None)38 if timer is None:39 logging.error('%s: stop timer recorded, but missing start timer!?',40 input_event.arg)41 if timer:42 assert input_event.name == timer[0]43 output_event = output_events.add()44 output_event.name = make_name(timer[0])45 output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis46 output_event.duration_milliseconds = (47 output_event.timestamp_milliseconds - timer[1])48 elif input_event.op == metrics.OP_NAMED_EVENT:49 output_event = output_events.add()50 output_event.name = make_name(input_event.name)51 output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis52 elif input_event.op == metrics.OP_GAUGE:53 output_event = output_events.add()54 output_event.name = make_name(input_event.name)55 output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis56 output_event.gauge = input_event.arg57 else:58 raise ValueError('unexpected op "%s" found in metric event: %s' % (59 input_event.op, input_event))60 # This is a sanity-check for unclosed timers.61 # TODO(wbbradley): Turn this back into an assert https://crbug.com/1001909.62 if timers:...
runner.py
Source:runner.py
1from ctypes import sizeof2import sys3from pykeymapper import InputEvent, CODE, TYPE, write_event, VALUE4def run_mapper(mapper):5 input_event = InputEvent()6 mapper = mapper()7 paused = False8 while sys.stdin.buffer.readinto(input_event) == sizeof(input_event):9 if input_event.type == TYPE.EV_MSC and input_event.code == CODE.MSC_SCAN:10 continue11 if input_event.type != TYPE.EV_KEY:12 write_event(input_event)13 continue14 if input_event.code == CODE.KEY_PAUSE and input_event.value == VALUE.KEY_DOWN:15 continue16 if input_event.code == CODE.KEY_PAUSE and input_event.value == VALUE.KEY_UP:17 paused = not paused18 continue19 if paused:20 write_event(input_event)21 continue...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run ATX automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
