Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use input_event method in ATX

Best Python code snippet using ATX

test_transformation_functions.py

Source:test_transformation_functions.py

1import unittest2import json3from managealooma.transformation_functions import add_column_based_on_null, add_columns_with_default, add_composite_key, \4    add_duplicate_fields, add_suffix, convert_all_event_fields_to_snake_case, convert_dictionary_fields_to_string, \5    convert_empty_value_to_none, convert_event_type_case, convert_null_to_zero, convert_spaces_and_special_characters_to_underscore, \6    convert_string_to_snake_case, convert_values_to_none, flatten_json, flatten_json_1_level, map_key_in_dictionary_to_value, \7    map_value_in_list_to_dictionary_key, mark_for_delete, parse_list_of_json_and_concat, remove_duplicate_field, remove_outer_key, \8    remove_starting_characters_from_keys, remove_whitespace, rename_fields, split_event_to_multiple_events, split_field_list_to_multiple_events, \9    whitelist_or_blacklist_columns10class TestTransformationFunctions(unittest.TestCase):11    # Test 112    def test_add_column_based_on_null_when_null(self):13        input = {'last_4': None}14        output = {'last_4': 'N'}15        self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null=None))16    # Test 217    def test_add_column_based_on_null_when_null_both_params_entered(self):18        input = {'last_4': None}19        output = {'last_4': 'N'}20        self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null='Y'))21    # Test 322    def test_add_column_based_on_null_when_not_null(self):23        input = {'last_4': 0000}24        output = {'last_4': 'Y'}25        self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null=None, new_value_if_not_null='Y'))26    # Test 427    def test_add_column_based_on_null_when_not_null_both_params_entered(self):28        input = {'last_4': 0000}29        output = {'last_4': 'Y'}30        self.assertEqual(output, add_column_based_on_null(event=input, field='last_4', new_field='last_4', new_value_if_null='N', new_value_if_not_null='Y'))31    # Test 532    def test_add_columns_with_default_add_columns(self):33        stuff_to_add = {'add_me': False, 'and_me': 123}34        input_event = {'just_one_key_to_start': 'stuff'}35        expected_output_event = {'add_me': False,36                                 'and_me': 123,37                                 'just_one_key_to_start': 'stuff'}38        self.assertEqual(expected_output_event, add_columns_with_default(input_event, field_and_default_dict=stuff_to_add))39    # Test 640    def test_add_columns_with_default_add_columns_skip_existing(self):41        stuff_to_add = {'add_me': False, 'skip_me': 123}42        input_event = {'just_one_key_to_start': 'stuff',43                       'skip_me': 'please do not change me'}44        expected_output_event = {'add_me': False,45                                 'skip_me': 'please do not change me',46                                 'just_one_key_to_start': 'stuff'}47        self.assertEqual(expected_output_event, add_columns_with_default(input_event, field_and_default_dict=stuff_to_add))48    # Test 749    def test_add_composite_key_key(self):50        input_event = {'product_id': 123456,51                       'product_name': 'my_product'}52        expected_output_event = {'product_id': 123456,53                                 'product_name': 'my_product',54                                 'product_id_name': '123456-my_product'}55        self.assertEqual(expected_output_event, add_composite_key(input_event, field_list=['product_id', 'product_name'], key_name='product_id_name'))56    # Test 857    def test_add_duplicate_fields_single(self):58        suffix_to_add = 'add'59        input_event = {'start_with_me': 123}60        expected_output_event = {'start_with_me': 123,61                                 'start_with_me_add': 123}62        self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffix_to_add, keep_original=True))63    # Test 964    def test_add_duplicate_fields_multiple(self):65        suffixes_to_add = ['add', 'add_me_too']66        input_event = {'start_with_me': 123}67        expected_output_event = {'start_with_me': 123,68                                 'start_with_me_add': 123,69                                 'start_with_me_add_me_too': 123}70        self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffixes_to_add, keep_original=True))71    # Test 1072    def test_add_duplicate_fields_multiple_remove_original(self):73        suffixes_to_add = ['add', 'add_me_too']74        input_event = {'start_with_me': 123}75        expected_output_event = {'start_with_me_add': 123,76                                 'start_with_me_add_me_too': 123}77        self.assertEqual(expected_output_event, add_duplicate_fields(input_event, field_name='start_with_me', suffix_or_suffix_list=suffixes_to_add, keep_original=False))78    # Test 1179    def test_add_suffix_single_field_default_separator(self):80        input_event = {'leave_me': 'my_name_stays',81                       'suffix_me': 'I want to be suffix_me_withthis'}82        expected_output_event = {'leave_me': 'my_name_stays',83                                 'suffix_me_withthis': 'I want to be suffix_me_withthis'}84        self.assertEqual(expected_output_event, add_suffix(input_event, fields='suffix_me', suffix='withthis'))85    # Test 1286    def test_add_suffix_two_fields_default_separator(self):87        input_event = {'leave_me': 'my_name_stays',88                       'suffix_me': 'I want to be suffix_me_withthis',89                       'suffix_me_too': 'I want to be suffix_me_withthis'}90        expected_output_event = {'leave_me': 'my_name_stays',91                                 'suffix_me_withthis': 'I want to be suffix_me_withthis',92                                 'suffix_me_too_withthis': 'I want to be suffix_me_withthis'}93        self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'suffix_me_too'], suffix='withthis'))94    # Test 1395    def test_add_suffix_single_field_new_separator(self):96        input_event = {'leave_me': 'my_name_stays',97                       'suffix_me': 'I want to be suffix_me---withthis'}98        expected_output_event = {'leave_me': 'my_name_stays',99                                 'suffix_me---withthis': 'I want to be suffix_me---withthis'}100        self.assertEqual(expected_output_event, add_suffix(input_event, fields='suffix_me', suffix='withthis', separator='---'))101    # Test 14102    def test_add_suffix_two_fields_new_separator(self):103        input_event = {'leave_me': 'my_name_stays',104                       'suffix_me': 'I want to be suffix_me---withthis',105                       'suffix_me_too': 'I want to be suffix_me---withthis'}106        expected_output_event = {'leave_me': 'my_name_stays',107                                 'suffix_me---withthis': 'I want to be suffix_me---withthis',108                                 'suffix_me_too---withthis': 'I want to be suffix_me---withthis'}109        self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'suffix_me_too'], suffix='withthis', separator='---'))110    # Test 15111    def test_add_suffix_field_not_in_event(self):112        input_event = {'leave_me': 'my_name_stays',113                       'suffix_me': 'I want to be suffix_me---withthis'}114        expected_output_event = {'leave_me': 'my_name_stays',115                                 'suffix_me---withthis': 'I want to be suffix_me---withthis'}116        self.assertEqual(expected_output_event, add_suffix(input_event, fields=['suffix_me', 'skip_me'], suffix='withthis', separator='---'))117    # Test 16118    def test_convert_all_event_fields_to_snake_case(self):119        input_event = {'MakeThisSnakeCase': 'stuff_in_field',120                       'Make This Snake Case': 'stuff_in_field',121                       'keep_this_snake_case': 'stuff_in_field',122                       'makeThisSnakeCase': 'stuff_in_field',123                       'leavemealone': 'left this one alone',124                       '_metadata': {'event_type': 'ObjectName',125                                     'input_label': 'Label',126                                     'input_type': 'salesforce'}}127        output_event = {'make_this_snake_case': 'stuff_in_field',128                        'make_this_snake_case': 'stuff_in_field',129                        'keep_this_snake_case': 'stuff_in_field',130                        'make_this_snake_case': 'stuff_in_field',131                        'leavemealone': 'left this one alone',132                        '_metadata': {'event_type': 'ObjectName',133                                      'input_label': 'Label',134                                      'input_type': 'salesforce'}}135        self.assertEqual(output_event, convert_all_event_fields_to_snake_case(input_event))136    # Test 17137    def test_convert_dictionary_fields_to_string_single_dictionary_is_string(self):138        input_event = {'dump me': {'make me': 'a string',139                                   'change me too': 'into - string'}}140        output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list='dump me')141        self.assertTrue(isinstance(output_event['dump me'], str))142    # Test 18143    def test_convert_dictionary_fields_to_string_single_dictionary_is_not_dict(self):144        input_event = {'dump me': {'make me': 'a string',145                                   'change me too': 'into - string'}}146        output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list='dump me')147        self.assertFalse(isinstance(output_event['dump me'], dict))148    # Test 19149    def test_convert_dictionary_fields_to_string_multiple_dictionary_is_string(self):150        input_event = {'dump me': {'make me': 'a string',151                                   'change me too': 'into - string'},152                       'dump me too': {'make me': 'a string',153                                       'change me too': 'into - string'}154                       }155        output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me', 'dump me too'])156        self.assertTrue(isinstance(output_event['dump me'], str) and isinstance(output_event['dump me too'], str))157    # Test 20158    def test_convert_dictionary_fields_to_string_multiple_dictionary_only_first_is_string(self):159        input_event = {'dump me': {'make me': 'a string',160                                   'change me too': 'into - string'},161                       'let me be': {'leave me': 'a dict',162                                     'i do not like': 'to change'}163                       }164        output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me'])165        self.assertTrue(isinstance(output_event['dump me'], str))166    # Test 21167    def test_convert_dictionary_fields_to_string_multiple_dictionary_not_specified_is_still_a_dict(self):168        input_event = {'dump me': {'make me': 'a string',169                                   'change me too': 'into - string'},170                       'let me be': {'leave me': 'a dict',171                                     'i do not like': 'to change'}172                       }173        output_event = convert_dictionary_fields_to_string(input_event, field_or_field_list=['dump me'])174        self.assertTrue(isinstance(output_event['let me be'], dict))175    # Test 22176    def test_convert_empty_value_to_none_list_to_null(self):177        input_event = {'empty_list': [],178                       'full_list': ['stuff', 'more_stuff']}179        expected_output_event = {'empty_list': None,180                                 'full_list': ['stuff', 'more_stuff']}181        self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_list'))182    # Test 23183    def test_convert_empty_value_to_none_list_string_to_null(self):184        input_event = {'empty_list': '[]',185                       'full_list': ['stuff', 'more_stuff']}186        expected_output_event = {'empty_list': None,187                                 'full_list': ['stuff', 'more_stuff']}188        self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_list'))189    # Test 24190    def test_convert_empty_value_to_none_dict_to_null(self):191        input_event = {'empty_dict': {},192                       'full_dict': {'a key': 'a value', 'another_key': 'a value'}}193        expected_output_event = {'empty_dict': None,194                                 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}195        self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_dict'))196    # Test 25197    def test_convert_empty_value_to_none_dict_string_to_null(self):198        input_event = {'empty_dict': '{}',199                       'full_dict': {'a key': 'a value', 'another_key': 'a value'}}200        expected_output_event = {'empty_dict': None,201                                 'full_dict': {'a key': 'a value', 'another_key': 'a value'}}202        self.assertEqual(expected_output_event, convert_empty_value_to_none(input_event, key_name='empty_dict'))203    # Test 26204    def test_convert_empty_value_to_none_string(self):205        input = {'winner': ''}206        expected_output = {'winner': None}207        self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))208    # Test 27209    def test_convert_empty_value_to_none_not_empty(self):210        input = {'winner': 'not empty'}211        expected_output = {'winner': 'not empty'}212        self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))213    # Test 28214    def test_convert_empty_value_to_none_string_with_spaces_only_to_none(self):215        input = {'winner': '    '}216        expected_output = {'winner': None}217        self.assertEqual(expected_output, convert_empty_value_to_none(input, 'winner'))218    # test 29219    def test_convert_event_type_case_single_event_force_upper(self):220        input_event = {'_metadata': {'event_type': 'my_schema.my_table'}}221        expected_output_event = {'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE'}}222        self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=True))223    # test 30224    def test_convert_event_type_case_list_event_force_upper(self):225        input_event = [{'_metadata': {'event_type': 'my_schema.my_table_one'}},226                       {'_metadata': {'event_type': 'My_schema.My_table_Two'}}]227        expected_output_event = [{'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_ONE'}},228                                 {'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_TWO'}}]229        self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=True))230    # test 31231    def test_convert_event_type_case_single_event_force_lower(self):232        input_event = {'_metadata': {'event_type': 'my_scheMA.my_table'}}233        expected_output_event = {'_metadata': {'event_type':  'my_schema.my_table'}}234        self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=False))235    # test 32236    def test_convert_event_type_case_list_event_force_lower(self):237        input_event = [{'_metadata': {'event_type': 'MY_SCHEMA.MY_TABLE_ONE'}},238                       {'_metadata': {'event_type': 'My_scheMA.MY_table_two'}}]239        expected_output_event = [{'_metadata': {'event_type': 'my_schema.my_table_one'}},240                                 {'_metadata': {'event_type': 'my_schema.my_table_two'}}]241        self.assertEqual(expected_output_event, convert_event_type_case(input_event, case_force_upper=False))242    # Test 33243    def test_convert_null_to_zero_list(self):244        input = {'a': None, 'b': None, 'c': 40, 'd': 20}245        expected_output = {'a': None, 'b': 0, 'c': 40, 'd': 20}246        self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list=['b', 'c', 'd']))247    # Test 34248    def test_convert_null_to_zero_single_field(self):249        input = {'a': None, 'b': None, 'c': 40, 'd': 20}250        expected_output = {'a': None, 'b': 0, 'c': 40, 'd': 20}251        self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list='b'))252    # Test 35253    def test_convert_null_to_zero_single_field_non_zero(self):254        input = {'a': None, 'b': None, 'c': 40, 'd': 20}255        expected_output = {'a': None, 'b': None, 'c': 40, 'd': 20}256        self.assertEqual(expected_output, convert_null_to_zero(input, field_or_field_list='c'))257    # Test 36258    def test_convert_spaces_and_special_characters_to_underscore(self):259        input_string = '$Scr "get-rid^-of-the@" special #characters%&space'260        output_string = '_scr__get_rid__of_the___special__characters__space'261        self.assertEqual(output_string, convert_spaces_and_special_characters_to_underscore(input_string))262    # Test 37263    def test_convert_string_to_snake_case_from_title_case_without_spaces(self):264        input_string = 'MakeThisSnakeCase'265        output_string = 'make_this_snake_case'266        self.assertEqual(output_string, convert_string_to_snake_case(input_string))267    # Test 38268    def test_convert_string_to_snake_case_from_title_case_with_spaces(self):269        input_string = 'Make This Snake Case'270        output_string = 'make_this_snake_case'271        self.assertEqual(output_string, convert_string_to_snake_case(input_string))272    # Test 39273    def test_convert_string_to_snake_case_already_snake_case(self):274        input_string = 'keep_this_snake_case'275        output_string = 'keep_this_snake_case'276        self.assertEqual(output_string, convert_string_to_snake_case(input_string))277    # Test 40278    def test_convert_string_to_snake_case_from_camel_case(self):279        input_string = 'makeThisSnakeCase'280        output_string = 'make_this_snake_case'281        self.assertEqual(output_string, convert_string_to_snake_case(input_string))282    # Test 41283    def test_convert_values_to_none_single_field_all_values(self):284        input_event = {'leave_alone': 1,285                       'change_to_none': 'NaN'}286        output_event = {'leave_alone': 1,287                        'change_to_none': None}288        self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none'))289    # Test 42290    def test_convert_values_to_none_multiple_fields_all_values(self):291        input_event = {'leave_alone': 1,292                       'change_to_none': 'NaN',293                       'change_me_too': 5.75}294        output_event = {'leave_alone': 1,295                        'change_to_none': None,296                        'change_me_too': None}297        self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list=['change_to_none', 'change_me_too']))298    # Test 43299    def test_convert_values_to_none_single_field_single_value_change(self):300        input_event = {'leave_alone': 1,301                       'change_to_none': 'NaN'}302        output_event = {'leave_alone': 1,303                        'change_to_none': None}304        self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none', field_values='NaN'))305    # Test 44306    def test_convert_values_to_none_single_field_single_value_dont_change(self):307        input_event = {'leave_alone': 1,308                       'change_to_none': 'Something Else'}309        output_event = {'leave_alone': 1,310                        'change_to_none': 'Something Else'}311        self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list='change_to_none', field_values='NaN'))312    # Test 45313    def test_convert_values_to_none_single_field_value_list_change(self):314        input_event = {'leave_alone': 1,315                       'change_to_none': 'NaN',316                       'also_change_to_none': 2}317        output_event = {'leave_alone': 1,318                        'change_to_none': None,319                        'also_change_to_none': None}320        self.assertEqual(output_event, convert_values_to_none(input_event, field_or_field_list=['change_to_none', 'also_change_to_none'], field_values=['NaN', 2]))321    # test 46322    def test_flatten_json_input_is_list_keep_original_true(self):323        input = {"batters": ['chocolate', 'strawberry', 'vanilla']}324        expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}325        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True)326        self.assertEqual(expected_output, output_event)327    # test 47328    def test_flatten_json_input_is_list_keep_original_false(self):329        input = {"batters": ['chocolate', 'strawberry', 'vanilla']}330        expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}331        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False)332        self.assertEqual(expected_output, output_event)333    # test 48334    def test_flatten_json_input_is_empty_dict_keep_original_false(self):335        input = {"batters": {}}336        expected_output = {}337        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False)338        self.assertEqual(expected_output, output_event)339    # test 49340    def test_flatten_json_input_is_empty_dict_keep_original_true(self):341        input = {"batters": {}}342        expected_output = {"batters": {}}343        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True)344        self.assertEqual(expected_output, output_event)345    # Test 50346    def test_flatten_json_from_string_1_level_keep_original_false_new_keep_as_json(self):347        input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}348        expected_output = {'cones_99 flake': '2.4',349                           'cones_cake': {},350                           'cones_sugar': [],351                           'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}}352        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=False)353        self.assertEqual(expected_output, output_event)354    # Test 51355    def test_flatten_json_from_string_1_level_keep_original_true_new_keep_as_json(self):356        input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}357        expected_output = {'cones': {'99 Flake': '2.4',358                                     'cake': {},359                                     'sugar': [],360                                     'waffle': {'chocolate': 'No',361                                                'vanilla': {'extra_charge': '2', 'sprinkles': 'Yes'}}},362                           'cones_99 flake': '2.4',363                           'cones_cake': {},364                           'cones_sugar': [],365                           'cones_waffle': {'chocolate': 'No',366                                            'vanilla': {'extra_charge': '2', 'sprinkles': 'Yes'}}}367        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True)368        self.assertEqual(expected_output, output_event)369    # Test 52370    def test_flatten_json_from_string_1_level_keep_original_false_dump_to_string(self):371        input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}372        expected_output = {'cones_99 flake': '2.4',373                           'cones_cake': '{}',374                           'cones_sugar': [],375                           'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}'}376        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=False, dump_to_string=True)377        self.assertEqual(expected_output, output_event)378    # Test 53379    def test_flatten_json_from_string_1_level_keep_original_true_dump_to_string(self):380        input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}381        expected_output = {'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',382                           'cones_99 flake': '2.4',383                           'cones_cake': '{}',384                           'cones_sugar': [],385                           'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}'}386        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=True)387        self.assertEqual(expected_output, output_event)388    # Test 54389    def test_flatten_json_from_dict_1_level_keep_original_false_dump_to_string(self):390        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},391                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},392                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}393        expected_output = {'batters_blueberry': '{"creaminess": "light", "sweetness": "super"}',394                           'batters_chocolate': '{"creaminess": "moderate", "sweetness": "very"}',395                           'batters_vanilla': '{"creaminess": "very", "sweetness": "medium"}'}396        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False, dump_to_string=True)397        self.assertEqual(expected_output, output_event)398    # Test 55399    def test_flatten_json_from_dict_1_level_keep_original_false_keep_as_json(self):400        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},401                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},402                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}403        expected_output = {'batters_blueberry': {"creaminess": "light", "sweetness": "super"},404                           'batters_chocolate': {"creaminess": "moderate", "sweetness": "very"},405                           'batters_vanilla': {"creaminess": "very", "sweetness": "medium"}}406        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=False, dump_to_string=False)407        self.assertEqual(expected_output, output_event)408    # Test 56409    def test_flatten_json_from_dict_1_level_keep_original_true_dump_to_string(self):410        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},411                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},412                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}413        expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '414                                      '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '415                                      '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',416                           'batters_blueberry': '{"creaminess": "light", "sweetness": "super"}',417                           'batters_chocolate': '{"creaminess": "moderate", "sweetness": "very"}',418                           'batters_vanilla': '{"creaminess": "very", "sweetness": "medium"}'}419        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=True)420        self.assertEqual(expected_output, output_event)421    # Test 57422    def test_flatten_json_from_dict_1_level_keep_original_true_keep_as_json(self):423        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},424                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},425                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}426        expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},427                                       "Chocolate": {"creaminess": "moderate", "sweetness": "very"},428                                       "Blueberry": {"creaminess": "light", "sweetness": "super"}},429                           'batters_blueberry': {"creaminess": "light", "sweetness": "super"},430                           'batters_chocolate': {"creaminess": "moderate", "sweetness": "very"},431                           'batters_vanilla': {"creaminess": "very", "sweetness": "medium"}}432        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)433        self.assertEqual(expected_output, output_event)434    # Test 58435    def test_flatten_json_from_dict_2_level_keep_original_false(self):436        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},437                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},438                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}439        expected_output = {'batters_blueberry_creaminess': 'light',440                           'batters_blueberry_sweetness': 'super',441                           'batters_chocolate_creaminess': 'moderate',442                           'batters_chocolate_sweetness': 'very',443                           'batters_vanilla_creaminess': 'very',444                           'batters_vanilla_sweetness': 'medium'}445        output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=False)446        self.assertEqual(expected_output, output_event)447    # Test 59448    def test_flatten_json_from_dict_2_level_keep_original_true_as_string(self):449        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},450                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},451                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}452        expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '453                                      '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '454                                      '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',455                           'batters_blueberry_creaminess': 'light',456                           'batters_blueberry_sweetness': 'super',457                           'batters_chocolate_creaminess': 'moderate',458                           'batters_chocolate_sweetness': 'very',459                           'batters_vanilla_creaminess': 'very',460                           'batters_vanilla_sweetness': 'medium'}461        output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=True, dump_to_string=True)462        self.assertEqual(expected_output, output_event)463    # Test 60464    def test_flatten_json_from_dict_2_level_keep_original_true_keep_as_json(self):465        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},466                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},467                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}468        expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},469                                       "Chocolate": {"creaminess": "moderate", "sweetness": "very"},470                                       "Blueberry": {"creaminess": "light", "sweetness": "super"}},471                           'batters_blueberry_creaminess': 'light',472                           'batters_blueberry_sweetness': 'super',473                           'batters_chocolate_creaminess': 'moderate',474                           'batters_chocolate_sweetness': 'very',475                           'batters_vanilla_creaminess': 'very',476                           'batters_vanilla_sweetness': 'medium'}477        output_event = flatten_json(input, field_or_field_list='batters', levels=2, keep_original=True, dump_to_string=False)478        self.assertEqual(expected_output, output_event)479    # Test 61480    def test_flatten_json_list_n_levels_2_fields(self):481        input = {482            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",483            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"484        }485        expected_output = {486            'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',487            'cones_99 flake': '2.4',488            'cones_cake': '{}',489            'cones_sugar': [],490            'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}',491            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",492            "ice_cream_chocolate": 1.99,493            "ice_cream_vanilla": 0.99}494        output_event = flatten_json(input, field_or_field_list=['cones', 'ice_cream'], levels=1,495                                    keep_original=True, dump_to_string=True)496        self.assertEqual(expected_output, output_event)497    # Test 62498    def test_flatten_json_from_string_3_level_keep_original_false_new(self):499        input = {"cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}500        expected_output = {'cones_99 flake': 2.4,501                           'cones_sugar': [],502                           'cones_waffle_chocolate': 'No',503                           'cones_waffle_vanilla_extra_charge': '2',504                           'cones_waffle_vanilla_sprinkles': 'Yes'}505        output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=False)506        self.assertEqual(expected_output, output_event)507    # Test 63508    def test_flatten_json_from_string_3_level_keep_original_true_new(self):509        input = {510            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}511        expected_output = {512            'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',513            'cones_99 flake': 2.4,514            'cones_sugar': [],515            'cones_waffle_chocolate': 'No',516            'cones_waffle_vanilla_extra_charge': '2',517            'cones_waffle_vanilla_sprinkles': 'Yes'518        }519        output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=True, dump_to_string=True)520        self.assertEqual(expected_output, output_event)521    # Test 64522    def test_flatten_json_capital_field(self):523        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},524                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},525                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}526        expected_output = {'batters': '{"Vanilla": {"creaminess": "very", "sweetness": "medium"}, '527                                      '"Chocolate": {"creaminess": "moderate", "sweetness": "very"}, '528                                      '"Blueberry": {"creaminess": "light", "sweetness": "super"}}',529                           'batters_blueberry_creaminess': 'light',530                           'batters_blueberry_sweetness': 'super',531                           'batters_chocolate_creaminess': 'moderate',532                           'batters_chocolate_sweetness': 'very',533                           'batters_vanilla_creaminess': 'very',534                           'batters_vanilla_sweetness': 'medium'}535        output_event = flatten_json(input, field_or_field_list='Batters', levels=2, keep_original=True, dump_to_string=True)536        self.assertEqual(expected_output, output_event)537    # Test 65538    def test_flatten_json_list_n_levels_1_field(self):539        input = {540            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",541            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"542        }543        expected_output = {544            'cones': '{"sugar":[],"cake":{},"99 Flake":"2.4","waffle":{"vanilla":{"sprinkles":"Yes","extra_charge":"2"},"chocolate":"No"}}',545            'cones_99 flake': '2.4',546            'cones_cake': '{}',547            'cones_sugar': [],548            'cones_waffle': '{"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}',549            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",550        }551        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=True)552        self.assertEqual(expected_output, output_event)553    # Test 66554    def test_flatten_json_from_string_3_level_keep_original_true_dump_string_false(self):555        input = {556            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}"}557        expected_output = {558            'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},559            'cones_99 flake': 2.4,560            'cones_sugar': [],561            'cones_waffle_chocolate': 'No',562            'cones_waffle_vanilla_extra_charge': '2',563            'cones_waffle_vanilla_sprinkles': 'Yes'564        }565        output_event = flatten_json(input, field_or_field_list='cones', levels=3, keep_original=True, dump_to_string=False)566        self.assertEqual(expected_output, output_event)567    # Test 67568    def test_flatten_json_input_is_list_keep_original_true_dump_string_false(self):569        input = {"batters": ['chocolate', 'strawberry', 'vanilla']}570        expected_output = {"batters": ['chocolate', 'strawberry', 'vanilla']}571        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)572        self.assertEqual(expected_output, output_event)573    # Test 68574    def test_flatten_json_input_is_empty_dict_keep_original_true_dump_string_false(self):575        input = {"batters": {}}576        expected_output = {"batters": {}}577        output_event = flatten_json(input, field_or_field_list='batters', levels=1, keep_original=True, dump_to_string=False)578        self.assertEqual(expected_output, output_event)579    # Test 69580    def test_flatten_json_capital_field_dump_string_false(self):581        input = {"batters": {"Vanilla": {"creaminess": "very", "sweetness": "medium"},582                             "Chocolate": {"creaminess": "moderate", "sweetness": "very"},583                             "Blueberry": {"creaminess": "light", "sweetness": "super"}}}584        expected_output = {'batters': {"Vanilla": {"creaminess": "very", "sweetness": "medium"},585                                       "Chocolate": {"creaminess": "moderate", "sweetness": "very"},586                                       "Blueberry": {"creaminess": "light", "sweetness": "super"}},587                           'batters_blueberry_creaminess': 'light',588                           'batters_blueberry_sweetness': 'super',589                           'batters_chocolate_creaminess': 'moderate',590                           'batters_chocolate_sweetness': 'very',591                           'batters_vanilla_creaminess': 'very',592                           'batters_vanilla_sweetness': 'medium'}593        output_event = flatten_json(input, field_or_field_list='Batters', levels=2, keep_original=True, dump_to_string=False)594        self.assertEqual(expected_output, output_event)595    # Test 70596    def test_flatten_json_list_n_levels_1_field_dump_string_false(self):597        input = {598            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",599            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"600        }601        expected_output = {602            'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},603            'cones_99 flake': '2.4',604            'cones_cake': {},605            'cones_sugar': [],606            'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"},607            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}",608        }609        output_event = flatten_json(input, field_or_field_list='cones', levels=1, keep_original=True, dump_to_string=False)610        self.assertEqual(expected_output, output_event)611    # Test 71612    def test_flatten_json_list_n_levels_2_fields_dump_string_false(self):613        input = {614            "cones": "{\"sugar\":[],\"cake\":{},\"99 Flake\":\"2.4\",\"waffle\":{\"vanilla\":{\"sprinkles\":\"Yes\",\"extra_charge\":\"2\"},\"chocolate\":\"No\"}}",615            "ice_cream": "{\"chocolate\":1.99,\"vanilla\":0.99}"616        }617        expected_output = {618            'cones': {"sugar": [], "cake": {}, "99 Flake": "2.4", "waffle": {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"}},619            'cones_99 flake': '2.4',620            'cones_cake': {},621            'cones_sugar': [],622            'cones_waffle': {"vanilla": {"sprinkles": "Yes", "extra_charge": "2"}, "chocolate": "No"},623            "ice_cream": {"chocolate": 1.99, "vanilla": 0.99},624            "ice_cream_chocolate": 1.99,625            "ice_cream_vanilla": 0.99}626        output_event = flatten_json(input, field_or_field_list=['cones', 'ice_cream'], levels=1,627                                    keep_original=True, dump_to_string=False)628        self.assertEqual(expected_output, output_event)629    # Test 72630    def test_flatten_json_1_level_and_dumps_using_ast(self):631        input_event = {'my_field': "{\"a\": None, \"b\": 11, \"c\": 3297, \"d\": 1497, \"e\": 11}"}632        output_event = {'my_field_a': None,633                        'my_field_b': 11,634                        'my_field_c': 3297,635                        'my_field_d': 1497,636                        'my_field_e': 11}637        self.assertEqual(output_event, flatten_json_1_level(event=input_event, field_name='my_field', field_name_underscore='my_field_', dump_to_string=True))638    # Test 73639    def test_map_key_in_dictionary_to_value_clean_transform(self):640        mapping_dict = {1: 'first',641                        2: 'second',642                        3: 'third'}643        input_event = {'number_field': 2,644                       '_metadata': {'event_type': 'event_type_name'}645                       }646        expected_output_event = {'number_field': 2,647                                 'number_field_with_name': 'second',648                                 '_metadata': {'event_type': 'event_type_name'}}649        self.assertEqual(expected_output_event, map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False))650    # Test 74651    def test_map_key_in_dictionary_to_value_null_allowed(self):652        mapping_dict = {1: 'first',653                        2: 'second',654                        3: 'third'}655        input_event = {'number_field': None,656                       '_metadata': {'event_type': 'event_type_name'}657                       }658        expected_output_event = {'number_field': None,659                                 'number_field_with_name': None,660                                 '_metadata': {'event_type': 'event_type_name'}}661        self.assertEqual(expected_output_event, map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=True))662    # Test 75663    def test_map_key_in_dictionary_to_value_missing_error(self):664        mapping_dict = {1: 'first',665                        2: 'second',666                        3: 'third'}667        input_event = {'number_field': 7,668                       '_metadata': {'event_type': 'event_type_name'}669                       }670        with self.assertRaises(BaseException) as cm:671            map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False)672        self.assertEqual(str(cm.exception), 'Missing enum transform event_type_name number_field')673    # Test 76674    def test_map_key_in_dictionary_to_value_value_null_error(self):675        mapping_dict = {1: 'first',676                        2: 'second',677                        3: 'third'}678        input_event = {'number_field': None,679                       '_metadata': {'event_type': 'event_type_name'}680                       }681        with self.assertRaises(BaseException) as cm:682            map_key_in_dictionary_to_value(input_event, mapping_dict, existing_column='number_field', new_column='number_field_with_name', allow_nulls=False)683        self.assertEqual(str(cm.exception), 'Missing enum transform event_type_name number_field')684    # Test 77685    def test_map_value_in_list_to_dictionary_key_successful_mapping(self):686        input_event = {"_metadata": {"event_type": "sweets"},687                       "type": "donut"}688        breakfast_item_categories = {"baked good": ["donut", "cake", "croissant"],689                                     "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}690        expected_output_event = {'_metadata': {'event_type': 'sweets'},691                                 'breakfast_item_category': 'baked good',692                                 'type': 'donut'}693        self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,694                                                                                    existing_column='type', new_column='breakfast_item_category',695                                                                                    allow_nulls=False, passthrough=True))696    # Test 78697    def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_false_passthrough_true(self):698        input_event = {"_metadata": {"event_type": "sweets"},699                       "type": "donut"}700        breakfast_item_categories = {"baked good": ["cake", "croissant"],701                                     "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}702        expected_output_event = {'_metadata': {'event_type': 'sweets'},703                                 'breakfast_item_category': 'donut',704                                 'type': 'donut'}705        self.assertEqual(expected_output_event,706                         map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,707                                                             existing_column='type', new_column='breakfast_item_category',708                                                             allow_nulls=False, passthrough=True))709    # Test 79710    def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_false_passthrough_false(self):711        input_event = {"_metadata": {"event_type": "sweets"},712                       "type": "donut",713                       }714        breakfast_item_categories = {"baked_goods": ["cake", "croissant"],715                                     "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}716        with self.assertRaises(Exception):717            map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,718                                                existing_column='type',719                                                new_column='breakfast_item_category',720                                                allow_nulls=False, passthrough=False)721    # Test 80722    def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_true_passthrough_true(self):723        input_event = {"_metadata": {"event_type": "sweets"},724                       "type": "donut"}725        breakfast_item_categories = {"baked good": ["cake", "croissant"],726                                     "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}727        expected_output_event = {'_metadata': {'event_type': 'sweets'},728                                 'breakfast_item_category': 'donut',729                                 'type': 'donut'}730        self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,731                                                                                    existing_column='type', new_column='breakfast_item_category',732                                                                                    allow_nulls=True, passthrough=True))733    # Test 81734    def test_map_value_in_list_to_dictionary_key_no_mapping_allow_nulls_true_passthrough_false(self):735        input_event = {"_metadata": {"event_type": "sweets"},736                       "type": "donut"}737        breakfast_item_categories = {"baked good": ["cake", "croissant"],738                                     "beverages": ["coffee", "tea", "orange juice", "grapefruit juice"]}739        expected_output_event = {'_metadata': {'event_type': 'sweets'},740                                 'breakfast_item_category': None,741                                 'type': 'donut'}742        self.assertEqual(expected_output_event, map_value_in_list_to_dictionary_key(input_event, mapping_dict_with_lists=breakfast_item_categories,743                                                                                    existing_column='type', new_column='breakfast_item_category',744                                                                                    allow_nulls=True, passthrough=False))745    # Test 82746    def test_mark_for_delete(self):747        test_mark_for_delete_input = """{"id": 13,748                                         "table_name": "alooma_test",749                                         "primary_key": "456789",750                                         "old_row_json": {"id":6,751                                                     "name":"User 6",752                                                     "created_at":"2017-08-23T05:01:51.753963",753                                                     "updated_at":"2017-08-26T15:27:13.455902"},754                                      "_metadata": {"event_type": "test"}}"""755        expected_output_event = {'_metadata': {'event_type': 'alooma_test',756                                               'table': 'alooma_test'},757                                 'created_at': '2017-08-23T05:01:51.753963',758                                 'id': 6,759                                 'mark_for_delete': True,760                                 'name': 'User 6',761                                 'updated_at': '2017-08-26T15:27:13.455902'}762        input_event = json.loads(test_mark_for_delete_input)763        input_event["old_row_json"] = json.dumps(input_event["old_row_json"])764        self.assertEqual(expected_output_event, mark_for_delete(input_event))765    # Test 83766    def test_parse_list_of_json_and_concat_keep_original_true(self):767        input_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},768                                         {'key_to_concat': 456, 'key_to_ignore': 'def'},769                                         {'key_to_concat': 789, 'key_to_ignore': 'ghi'}]}770        expected_output_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},771                                                   {'key_to_concat': 456, 'key_to_ignore': 'def'},772                                                   {'key_to_concat': 789, 'key_to_ignore': 'ghi'}],773                                 'list_of_dicts_key_to_concats': [123, 456, 789]}774        self.assertEqual(expected_output_event, parse_list_of_json_and_concat(input_event, field_name='list_of_dicts', keep_original=True, field_to_keep='key_to_concat'))775    # Test 84776    def test_parse_list_of_json_and_concat_keep_original_false(self):777        input_event = {'list_of_dicts': [{'key_to_concat': 123, 'key_to_ignore': 'abc'},778                                         {'key_to_concat': 456, 'key_to_ignore': 'def'},779                                         {'key_to_concat': 789, 'key_to_ignore': 'ghi'}]}780        expected_output_event = {'list_of_dicts_key_to_concats': [123, 456, 789]}781        self.assertEqual(expected_output_event, parse_list_of_json_and_concat(input_event, field_name='list_of_dicts', keep_original=False, field_to_keep='key_to_concat'))782    # Test 85783    def test_remove_duplicate_field_remove(self):784        input_event = {'Im Key': 'I am repeated information',785                       'im_key': 'I am repeated information'}786        expected_output_event = {'Im Key': 'I am repeated information'}787        self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='Im Key', field_to_discard='im_key'))788    # Test 86789    def test_remove_duplicate_field_remove_keep_field_none(self):790        input_event = {'Im Key': None,791                       'im_key': 'I should be kept'}792        expected_output_event = {'im_key': 'I should be kept'}793        self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='Im Key', field_to_discard='im_key'))794    # Test 87795    def test_remove_duplicate_field_missing_a_field_no_discard(self):796        input_event = {'im useless': None,797                       'keep_me': 'I should be kept'}798        expected_output_event = {'im useless': None,799                                 'keep_me': 'I should be kept'}800        self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='keep_me', field_to_discard='discard_me'))801    # Test 88802    def test_remove_duplicate_field_missing_a_field_keep_discard(self):803        input_event = {'im useless': None,804                       'discard_me': 'I stay Anyway'}805        expected_output_event = {'im useless': None,806                                 'discard_me': 'I stay Anyway'}807        self.assertEqual(expected_output_event, remove_duplicate_field(input_event, field_to_keep='keep_me', field_to_discard='discard_me'))808    # Test 89809    def test_remove_outer_key_remove(self):810        input_event = {'a key': 'stuff',811                       'nested_stuff': {'field_one': 1,812                                        'field_two': 2,813                                        'field_three': 3}814                       }815        expected_output_event = {'a key': 'stuff',816                                 'field_one': 1,817                                 'field_two': 2,818                                 'field_three': 3819                                 }820        self.assertEqual(expected_output_event, remove_outer_key(input_event, key_name='nested_stuff'))821    # Test 90822    def test_remove_outer_key_no_dict(self):823        input_event = {'a key': 'stuff',824                       'nested_stuff': {'field_one': 1,825                                        'field_two': 2,826                                        'field_three': 3}827                       }828        expected_output_event = {'a key': 'stuff',829                                 'nested_stuff': {'field_one': 1,830                                                  'field_two': 2,831                                                  'field_three': 3}832                                 }833        self.assertEqual(expected_output_event, remove_outer_key(input_event, key_name='a key'))834    # Test 91835    def test_remove_outer_key_has_key(self):836        input = {'remove_me': {'i want to be top level': 'keep me',837                               'i want to be top level too': 'keep me too'}}838        expected_output = {'i want to be top level': 'keep me',839                           'i want to be top level too': 'keep me too'}840        self.assertEqual(expected_output, remove_outer_key(input, key_name='remove_me'))841    # Test 92842    def test_remove_outer_key_missing_key(self):843        input = {'remove_me': {'i want to be top level': 'keep me',844                               'i want to be top level too': 'keep me too'}}845        expected_output = {'remove_me': {'i want to be top level': 'keep me',846                                         'i want to be top level too': 'keep me too'}}847        self.assertEqual(expected_output, remove_outer_key(input, key_name='i_dont_exist'))848    # Test 93849    def test_remove_outer_key_has_key_and_nest(self):850        input = {'remove_me': {'i want to be top level': 'keep me',851                               'i want to be top level too': 'keep me too',852                               'My children stay nested': {'one down': 'mr one', 'two down': 'ms two', 'has more nest': {'way down': 'mr way', 'way down again': 'ms way'}}}}853        expected_output = {'i want to be top level': 'keep me',854                           'i want to be top level too': 'keep me too',855                           'My children stay nested': {'one down': 'mr one', 'two down': 'ms two', 'has more nest': {'way down': 'mr way', 'way down again': 'ms way'}}}856        self.assertEqual(expected_output, remove_outer_key(input, key_name='remove_me'))857    # Test 94858    def test_remove_starting_characters_from_keys_one_char(self):859        input_event = {'_metadata': {},860                       '$strip_me': 'no more $ in key',861                       '$strip_me_too': 'no more $ in key',862                       '_dont_strip': 'leave my underscore',863                       'nothing_here': 'nothing special'}864        expected_output_event = {'_metadata': {},865                                 '_dont_strip': 'leave my underscore',866                                 'nothing_here': 'nothing special',867                                 'strip_me_too': 'no more $ in key',868                                 'strip_me': 'no more $ in key'}869        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$'))870    # Test 95871    def test_remove_starting_characters_from_keys_many_char(self):872        input_event = {'_metadata': {},873                       'MY$-strip_me': 'no more MY$- in key',874                       'MY$-strip_me_too': 'no more MY$- in key',875                       '_dont_strip': 'leave_my_underscore',876                       'nothing_here': 'nothing_special'}877        expected_output_event = {'_metadata': {},878                                 '_dont_strip': 'leave_my_underscore',879                                 'nothing_here': 'nothing_special',880                                 'strip_me': 'no more MY$- in key',881                                 'strip_me_too': 'no more MY$- in key'}882        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='MY$-'))883    # Test 96884    def test_remove_starting_characters_from_keys_no_keys_present(self):885        input_event = {'_metadata': {},886                       'nothing_here': 'nothing to strip',887                       'nothing_here_either': 'nothing to strip'}888        expected_output_event = {'_metadata': {},889                                 'nothing_here': 'nothing to strip',890                                 'nothing_here_either': 'nothing to strip'}891        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$'))892    # Test 97893    def test_remove_starting_characters_from_keys_starts_with_part(self):894        input_event = {'_metadata': {},895                       '$$strip_me': 'starts with two - strimp em',896                       '$strip_me': 'starts with just one - stays here'}897        expected_output_event = {'_metadata': {},898                                 'strip_me': 'starts with two - strimp em',899                                 '$strip_me': 'starts with just one - stays here'}900        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$$'))901    # Test 98902    def test_remove_starting_characters_from_sub_field(self):903        input_event = {'$strip_my_subs': {'$strip_me': 'strimp em',904                                          '$and_me': 'strip em too',905                                          'not_me': 'I never change'}}906        expected_output_event = {'$strip_my_subs': {'strip_me': 'strimp em',907                                                    'and_me': 'strip em too',908                                                    'not_me': 'I never change'}}909        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$', field_with_json='$strip_my_subs'))910    # Test 99911    def test_remove_starting_characters_from_sub_field_missing_key(self):912        input_event = {'$strip_my_subs': {'$strip_me': 'strimp em',913                                          '$and_me': 'strip em too',914                                          'not_me': 'I never change'}}915        expected_output_event = {'$strip_my_subs': {'$strip_me': 'strimp em',916                                                    '$and_me': 'strip em too',917                                                    'not_me': 'I never change'}}918        self.assertEqual(expected_output_event, remove_starting_characters_from_keys(input_event, starting_characters='$', field_with_json='not_in_event'))919    # Test 100920    def test_trim_whitespace(self):921        input = {'strip': '  this is my example               '}922        output = {'strip': 'this is my example'}923        self.assertEqual(output, remove_whitespace(input, field_or_field_list='strip'))924    # Test 101925    def test_remove_whitespace_list(self):926        input = {'strip': '  this is my example               ',927                 'strip me too': '  this is my example'}928        output = {'strip': 'this is my example',929                  'strip me too': 'this is my example'}930        self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip', 'strip me too']))931    # Test 102932    def test_remove_whitespace_only_specifiy_one(self):933        input = {'strip': '  this is my example               ',934                 'strip me too': '  this is my example'}935        output = {'strip': 'this is my example',936                  'strip me too': '  this is my example'}937        self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip']))938    # Test 103939    def test_remove_whitespace_has_an_int(self):940        input = {'strip': 1}941        output = {'strip': 1}942        self.assertEqual(output, remove_whitespace(input, field_or_field_list=['strip']))943    # Test 104944    def test_rename_fields_one_field_in_event(self):945        input_event = {'leave_me': 'my_name_stays',946                       'change_me': 'My new name is: i_changed'}947        expected_output_event = {'leave_me': 'my_name_stays',948                                 'i_changed': 'My new name is: i_changed'}949        self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed'}))950    # Test 105951    def test_rename_fields_one_field_not_in_event(self):952        input_event = {'leave_me': 'my_name_stays',953                       'change_me': 'My new name is: i_changed'}954        expected_output_event = {'leave_me': 'my_name_stays',955                                 'change_me': 'My new name is: i_changed'}956        self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'skip_me': 'not_in_event'}))957    # Test 106958    def test_rename_fields_two_fields_in_event(self):959        input_event = {'leave_me': 'my_name_stays',960                       'change_me': 'My new name is: i_changed',961                       'change_me_too': 'My new name is: i_changed_too'}962        expected_output_event = {'leave_me': 'my_name_stays',963                                 'i_changed': 'My new name is: i_changed',964                                 'i_changed_too': 'My new name is: i_changed_too'}965        self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed', 'change_me_too': 'i_changed_too'}))966    # Test 107967    def test_rename_fields_two_fields_one_in_event(self):968        input_event = {'leave_me': 'my_name_stays',969                       'change_me': 'My new name is: i_changed'}970        expected_output_event = {'leave_me': 'my_name_stays',971                                 'i_changed': 'My new name is: i_changed'}972        self.assertEqual(expected_output_event, rename_fields(input_event, field_dict={'change_me': 'i_changed', 'skip_me': 'not_in_event'}))973    # Test 108974    def test_rename_fields_raise_exception(self):975        rename_field_dict = {'im_similar': 'properties_user_id',976                             'imsimilar': 'properties_user_id'}977        input_event = {'im_similar': 2,978                       'imsimilar': 3}979        with self.assertRaises(BaseException):980            rename_fields(input_event, rename_field_dict)981    # Test 109982    def test_split_event_to_multiple_events_skip_for_parent_id(self):983        input_event = {'_metadata': {'event_type': 'schema_name.event_name',984                                     '@parent_uuid': '123'},985                       'a_non_metadata_field': 'some value'}986        expected_output_event = {'_metadata': {'event_type': 'schema_name.event_name',987                                               '@parent_uuid': '123'},988                                 'a_non_metadata_field': 'some value'}989        self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['one', 'two']))990    # Test 110991    def test_split_event_to_multiple_events_split_into_two(self):992        input_event = {'_metadata': {'event_type': 'schema_name.event_name',993                                     '@parent_uuid': ''},994                       'a_non_metadata_field': 'some value'}995        expected_output_event = [{'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_one'},996                                  'a_non_metadata_field': 'some value'},997                                 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_two'},998                                  'a_non_metadata_field': 'some value'}]999        self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two']))1000    # Test 1111001    def test_split_event_to_multiple_events_split_into_three(self):1002        input_event = {'_metadata': {'event_type': 'schema_name.event_name',1003                                     '@parent_uuid': ''},1004                       'a_non_metadata_field': 'some value'}1005        expected_output_event = [{'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_one'},1006                                  'a_non_metadata_field': 'some value'},1007                                 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_two'},1008                                  'a_non_metadata_field': 'some value'},1009                                 {'_metadata': {'@parent_uuid': '', 'event_type': 'schema_name.table_three'},1010                                  'a_non_metadata_field': 'some value'}]1011        self.assertEqual(expected_output_event, split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two', 'table_three']))1012    # Test 1121013    def test_split_event_to_multiple_events_throw_exception_if_not_fully_qualified(self):1014        input_event = {'_metadata': {'event_type': 'schema_name_event_name',1015                                     '@parent_uuid': ''},1016                       'a_non_metadata_field': 'some value'}1017        with self.assertRaises(BaseException) as cm:1018            split_event_to_multiple_events(event=input_event, table_name_list=['table_one', 'table_two', 'table_three'])1019        self.assertEqual(str(cm.exception), 'Only fully qualified events can be split. Event type must be schema_name.table_name')1020    # Test 1131021    def test_split_field_list_to_multiple_events_split(self):1022        input = {'id': 1,1023                 'names': ['first', 'second'],1024                 '_metadata': {'uuid': '1a'}}1025        output = [{'id': 1,1026                   'name': 'first',1027                   'counter': 1,1028                   '_metadata': {'uuid': '1a'}},1029                  {'id': 1,1030                   'name': 'second',1031                   'counter': 2,1032                   '_metadata': {'uuid': '1a'}}1033                  ]1034        self.assertEqual(output, split_field_list_to_multiple_events(event=input, fields_to_split=['names'], add_counter=True, counter_name='counter', reverse=False))1035    # Test 1141036    def test_split_field_list_to_multiple_events_skip_for_parentd_uuid(self):1037        input = {'id': 1,1038                 'names': ['first', 'second'],1039                 '_metadata': {'@parent_uuid': '1a'}}1040        output = {'id': 1,1041                  'names': ['first', 'second'],1042                  '_metadata': {'@parent_uuid': '1a'}}1043        self.assertEqual(output, split_field_list_to_multiple_events(event=input, fields_to_split=['names'], add_counter=True, counter_name='counter', reverse=False))1044    # Test 1151045    def test_whitelist_or_blacklist_columns_whitelist(self):1046        input = {"Cookies": None,1047                 "cake": None,1048                 "brownies": "yum",1049                 "iceCream": "mmmm",1050                 "_metadata": {"table": "sweets"}}1051        expected_output = {"Cookies": None,1052                           "cake": None,1053                           "iceCream": "mmmm",1054                           "_metadata": {"table": "sweets"}}1055        self.assertEqual(expected_output, whitelist_or_blacklist_columns(input, field_list=['cookies', 'Cake', 'IceCream'], white_or_black_list='whitelist'))1056    # Test 1161057    def test_whitelist_or_blacklist_columns_blacklist(self):1058        input = {"Cookies": None,1059                 "cake": None,1060                 "brownies": "yum",1061                 "iceCream": "mmmm",1062                 "_metadata": {"table": "sweets"}}1063        expected_output = {"brownies": "yum",1064                           "_metadata": {"table": "sweets"}}...

metrics.py

Source:metrics.py

1# -*- coding: utf-8 -*-2# Copyright 2019 The Chromium OS Authors. All rights reserved.3# Use of this source code is governed by a BSD-style license that can be4# found in the LICENSE file.5"""Metrics for general consumption.6See infra/proto/metrics.proto for a description of the type of record that this7module will be creating.8"""9from __future__ import print_function10import sys11from chromite.lib import cros_logging as logging12from chromite.utils import metrics13assert sys.version_info >= (3, 6), 'This module requires Python 3.6+'14def deserialize_metrics_log(output_events, prefix=None):15  """Read the current metrics events, adding to output_events.16  This layer facilitates converting between the internal17  chromite.utils.metrics representation of metric events and the18  infra/proto/src/chromiumos/metrics.proto output type.19  Args:20    output_events: A chromiumos.MetricEvent protobuf message.21    prefix: A string to prepend to all metric event names.22  """23  timers = {}24  def make_name(name):25    """Prepend a closed-over prefix to the given name."""26    if prefix:27      return '%s.%s' % (prefix, name)28    else:29      return name30  # Reduce over the input events to append output_events.31  for input_event in metrics.read_metrics_events():32    if input_event.op == metrics.OP_START_TIMER:33      timers[input_event.arg] = (input_event.name,34                                 input_event.timestamp_epoch_millis)35    elif input_event.op == metrics.OP_STOP_TIMER:36      # TODO(wbbradley): Drop the None fallback https://crbug.com/1001909.37      timer = timers.pop(input_event.arg, None)38      if timer is None:39        logging.error('%s: stop timer recorded, but missing start timer!?',40                      input_event.arg)41      if timer:42        assert input_event.name == timer[0]43        output_event = output_events.add()44        output_event.name = make_name(timer[0])45        output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis46        output_event.duration_milliseconds = (47            output_event.timestamp_milliseconds - timer[1])48    elif input_event.op == metrics.OP_NAMED_EVENT:49      output_event = output_events.add()50      output_event.name = make_name(input_event.name)51      output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis52    elif input_event.op == metrics.OP_GAUGE:53      output_event = output_events.add()54      output_event.name = make_name(input_event.name)55      output_event.timestamp_milliseconds = input_event.timestamp_epoch_millis56      output_event.gauge = input_event.arg57    else:58      raise ValueError('unexpected op "%s" found in metric event: %s' % (59          input_event.op, input_event))60  # This is a sanity-check for unclosed timers.61  # TODO(wbbradley): Turn this back into an assert https://crbug.com/1001909.62  if timers:...

runner.py

Source:runner.py

1from ctypes import sizeof2import sys3from pykeymapper import InputEvent, CODE, TYPE, write_event, VALUE4def run_mapper(mapper):5    input_event = InputEvent()6    mapper = mapper()7    paused = False8    while sys.stdin.buffer.readinto(input_event) == sizeof(input_event):9        if input_event.type == TYPE.EV_MSC and input_event.code == CODE.MSC_SCAN:10            continue11        if input_event.type != TYPE.EV_KEY:12            write_event(input_event)13            continue14        if input_event.code == CODE.KEY_PAUSE and input_event.value == VALUE.KEY_DOWN:15            continue16        if input_event.code == CODE.KEY_PAUSE and input_event.value == VALUE.KEY_UP:17            paused = not paused18            continue19        if paused:20            write_event(input_event)21            continue...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.