How to use merge_shards method in localstack

Best Python code snippet using localstack_python

transformer.py

Source:transformer.py

...136                label=int(cancer),137                size=size,138                threads=threads,139                dataset=dataset)140def merge_shards(in_files, fname):141    with tf.python_io.TFRecordWriter(fname, options=OPTIONS) as writer:142        for in_file in in_files:143            for record in tf.python_io.tf_record_iterator(in_file, options=OPTIONS):144                writer.write(record)145def merge_shards_in_folder(transformed_folder, transformed_out, size=(256, 256)):146    healthy = []147    cancer = []148    for file in os.listdir(transformed_folder):149        path = os.path.join(transformed_folder, file)150        if not file.startswith('label') or not file.endswith('tfrecord'):151            continue152        if file.startswith('label_1'):153            cancer.append(path)154        elif file.startswith('label_0'):155            healthy.append(path)156        else:157            raise ValueError('Invalid file: ' + str(path))158    merge_shards(healthy, os.path.join(transformed_out, 'healthy.tfrecord'))159    merge_shards(cancer, os.path.join(transformed_out, 'cancer.tfrecord'))160def load_inbreast_mask(mask_path, imshape=(4084, 3328)):161    """162    This function loads a osirix xml region as a binary numpy array for INBREAST163    dataset164    @mask_path : Path to the xml file165    @imshape : The shape of the image as an array e.g. [4084, 3328]166    return: numpy array where positions in the roi are assigned a value of 1.167    """168    def load_point(point_string):169        x, y = tuple([float(num) for num in point_string.strip('()').split(',')])170        return y, x171    mask_shape = np.transpose(imshape)172    mask = np.zeros(mask_shape)173    with open(mask_path, 'rb') as mask_file:...

test_inserts.py

Source:test_inserts.py

...31        table.add(None)32        table.add("test")33        table.add({})3435        table.merge_shards()36        result = jx.sort(table.all_records(), ".")3738        expected = [42, "test", NULL, NULL]3940        self.assertEqual(result, expected)4142    def test_array(self):43        dataset = self.dataset44        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)45        table.add({"b": [1, 2, 3, 4, 5, 6]})46        table.merge_shards()47        result = jx.sort(table.all_records(), "a")4849        expected = [{"b": [1, 2, 3, 4, 5, 6]}]50        self.assertEqual(result, expected)5152    def test_one_then_many_then_merge(self):53        dataset = self.dataset54        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)55        table.add({"a": 1, "b": {"c": 1, "d": 1}})56        table.add({"a": 2, "b": [{"c": 1, "d": 1}, {"c": 2, "d": 2}]})5758        table.merge_shards()59        result = jx.sort(table.all_records(), "a")6061        expected = [62            {"a": 1, "b": {"c": 1, "d": 1}},63            {"a": 2, "b": [{"c": 1, "d": 1}, {"c": 2, "d": 2}]},64        ]6566        self.assertEqual(result, expected)6768    def test_one_then_deep_arrays1(self):69        dataset = self.dataset70        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)71        table.add({"a": 1, "b": {"c": [{"e": "e"}, {"e": 1}]}})72        table.add({"a": 2, "b": {"c": 1}})7374        table.merge_shards()75        result = jx.sort(table.all_records(), "a")7677        expected = [78            {"a": 1, "b": {"c": [{"e": "e"}, {"e": 1}]}},79            {"a": 2, "b": {"c": 1}},80        ]8182        self.assertEqual(result, expected)8384    def test_one_then_deep_arrays2(self):85        dataset = self.dataset86        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)87        table.add({"a": 1, "b": {"c": [{"e": "e"}, {"e": 1}]}})88        table.add({"a": 2, "b": {"c": 1}})89        table.add({"a": 3, "b": [{"c": [{"e": 2}, {"e": 3}]}, {"c": 42}]})9091        table.merge_shards()92        result = jx.sort(table.all_records(), "a")9394        expected = [95            {"a": 1, "b": {"c": [{"e": "e"}, {"e": 1}]}},96            {"a": 2, "b": {"c": 1}},97            {"a": 3, "b": [{"c": [{"e": 2}, {"e": 3}]}, {"c": 42}]},98        ]99100        self.assertEqual(result, expected)101102    def test_inject_arrays(self):103        dataset = self.dataset104        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)105106        table.add({"a": 1, "b": {"c": {"e": 42}}})107        table.add({"a": 3, "b": [{"c": {"e": 2}}, {"c": {"e": 3}}]})108109        table.merge_shards()110        result = jx.sort(table.all_records(), "a")111112        expected = [113            {"a": 1, "b": {"c": {"e": 42}}},114            {"a": 3, "b": [{"c": {"e": 2}}, {"c": {"e": 3}}]},115        ]116117        self.assertEqual(result, expected)118119    def test_has_arrays(self):120        dataset = self.dataset121        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)122123        table.add({"a": 1, "b": {"c": {"e": 42}}})124        table.add({"a": 3, "b": [{"c": {"e": 2}}, {"c": {"e": 3}, "f": 42}]})125126        table.merge_shards()127        result = jx.sort(table.all_records(), "a")128129        expected = [130            {"a": 1, "b": {"c": {"e": 42}}},131            {"a": 3, "b": [{"c": {"e": 2}}, {"c": {"e": 3}}]},132        ]133134        self.assertEqual(result, expected)135136    def test_zero_array(self):137        dataset = self.dataset138        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)139140        table.add({"a": 3, "b": []})141142        table.merge_shards()143        result = jx.sort(table.all_records(), "a")144145        expected = [146            {"a": 3, "b": NULL},147        ]148149        self.assertEqual(result, expected)150151    def test_null(self):152        dataset = self.dataset153        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)154155        table.add({"a": 3, "b": None})156157        table.merge_shards()158        result = jx.sort(table.all_records(), "a")159160        expected = [161            {"a": 3, "b": NULL},162        ]163164        self.assertEqual(result, expected)165166    def test_infinity(self):167        dataset = self.dataset168        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)169170        table.add({"a": 3, "b": -math.inf})171172        table.merge_shards()173        result = jx.sort(table.all_records(), "a")174175        expected = [176            {"a": 3, "b": NULL},177        ]178179        self.assertEqual(result, expected)180181    @skipIf(PY2, "no enums for python 2")182    def test_enum(self):183        class Status(Enum):184            PASS = 0185            FAIL = 1186            INTERMITTENT = 2187188        dataset = self.dataset189        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)190191        table.add({"a": Status.PASS})192193        table.merge_shards()194        result = jx.sort(table.all_records(), "a")195196        expected = [197            {"a": "PASS"},198        ]199200        self.assertEqual(result, expected)201202    def test_encoding_on_deep_arrays(self):203        dataset = self.dataset204        table = dataset.create_or_replace_table(table=tests.table_name(), sharded=True)205        table.add({"__a": 1, "__b": {"__c": [{"__e": "e"}, {"__e": 1}]}})206        table.add({"__a": 2, "__b": {"__c": 1}})207        table.add({"__a": 3, "__b": [{"__c": [{"__e": 2}, {"__e": 3}]}]})208209        table.merge_shards()210        result = jx.sort(table.all_records(), "__a")211212        expected = [213            {"__a": 1, "__b": {"__c": [{"__e": "e"}, {"__e": 1}]}},214            {"__a": 2, "__b": {"__c": 1}},215            {"__a": 3, "__b": {"__c": [{"__e": 2}, {"__e": 3}]}},216        ]217218        self.assertEqual(result, expected)219220    def test_top_level_field_order(self):221        dataset = self.dataset222223        table = dataset.create_or_replace_table(224            table=tests.table_name(),225            sharded=True,226            schema={"push": {"id": {"_i_": "integer"}}, "etl": {"timestamp": {"_t_": "time"}}},227            # REVERSE ALPHABTICAL ORDER228            top_level_fields=OrderedDict([229                ("push", {"id": "_push_id"}),230                ("etl", {"timestamp": "_etl_timestamp"})231            ]),232        )233        today = Date.today()234        now = Date.now()235        table.add({"push": {"id": 1}, "etl": {"timestamp": today}, "a": 1})236        table.add({"push": {"id": 2}, "etl": {"timestamp": now}, "b": 2})237238        table.merge_shards()239        result = jx.sort(table.all_records(), "push.id")240241        expected = [242            {"push": {"id": 1}, "etl": {"timestamp": today}, "a": 1},243            {"push": {"id": 2}, "etl": {"timestamp": now}, "b": 2},244        ]245
...

merge.py

Source:merge.py

2from mo_logs import startup, constants, Log3def merge(config):4    container = bigquery.Dataset(config.destination)5    index = container.get_or_create_table(config.destination)6    index.merge_shards()7def main():8    try:9        config = startup.read_settings()10        constants.set(config.constants)11        Log.start(config.debug)12        merge(config.push)13    except Exception as e:14        Log.error("Problem with etl", cause=e)15    finally:16        Log.stop()17if __name__ == "__main__":...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.