How to use test_good_path method in green

Best Python code snippet using green

test_operator_decoders_image.py

Source:test_operator_decoders_image.py

1# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14from nvidia.dali import Pipeline, pipeline_def15import nvidia.dali.fn as fn16import nvidia.dali.ops as ops17import nvidia.dali.types as types18import math19import os20import random21import numpy as np22import glob23from nose import SkipTest24from nose.tools import assert_raises25from test_utils import check_batch26from test_utils import compare_pipelines27from test_utils import RandomDataIterator28from test_utils import get_dali_extra_path29from test_utils import check_output_pattern30from test_utils import to_array31def get_img_files(data_path, subdir='*', ext=None):32    if subdir is None:33        subdir = ''34    if ext:35        if isinstance(ext, (list, tuple)):36            files = []37            for e in ext:38                files += glob.glob(data_path + f"/{subdir}/*.{e}")39        else:40            files = glob.glob(data_path + f"/{subdir}/*.{ext}")41        return files42    else:43        files = glob.glob(data_path  + f"/{subdir}/*.*")44        txt_files = glob.glob(data_path  + f"/{subdir}/*.txt")45        return list(set(files) - set(txt_files))46@pipeline_def47def decoder_pipe(data_path, device, use_fast_idct=False, memory_stats=False):48    inputs, labels = fn.readers.file(file_root = data_path,49                                     shard_id = 0,50                                     num_shards = 1,51                                     name="Reader")52    decoded = fn.decoders.image(inputs, device = device, output_type = types.RGB, use_fast_idct=use_fast_idct,53                                memory_stats=memory_stats)54    return decoded, labels55test_data_root = get_dali_extra_path()56good_path = 'db/single'57missnamed_path = 'db/single/missnamed'58test_good_path = {'jpeg', 'mixed', 'png', 'tiff', 'pnm', 'bmp', 'jpeg2k'}59test_missnamed_path = {'jpeg', 'png', 'tiff', 'pnm', 'bmp'}60def run_decode(data_path, batch, device, threads, memory_stats=False):61    pipe = decoder_pipe(data_path=data_path, batch_size=batch, num_threads=threads, device_id=0, device=device, memory_stats=memory_stats, prefetch_queue_depth=1)62    pipe.build()63    iters = math.ceil(pipe.epoch_size("Reader") / batch)64    for _ in range(iters):65        pipe.run()66def test_image_decoder():67    def log(img_type, size, device, threads):68        pass69    for device in {'cpu', 'mixed'}:70        for batch_size in {1, 10}:71            for img_type in test_good_path:72                for threads in {1, random.choice([2, 3, 4])}:73                    data_path = os.path.join(test_data_root, good_path, img_type)74                    run_decode(data_path, batch_size, device, threads)75                    yield log, img_type, batch_size, device, threads76            for img_type in test_missnamed_path:77                for threads in {1, random.choice([2, 3, 4])}:78                    data_path = os.path.join(test_data_root, missnamed_path, img_type)79                    run_decode(data_path, batch_size, device, threads)80                    yield log, img_type, batch_size, device, threads81# TODO(januszl): check padding behavior82@pipeline_def83def create_decoder_slice_pipeline(data_path, device):84    jpegs, _ = fn.readers.file(file_root = data_path,85                               shard_id = 0,86                               num_shards = 1,87                               name = "Reader")88    anchor = fn.random.uniform(range=[0.05, 0.15], shape=(2,))89    shape = fn.random.uniform(range=[0.5, 0.7], shape=(2,))90    images_sliced_1 = fn.decoders.image_slice(jpegs,91                                              anchor,92                                              shape,93                                              device = device,94                                              hw_decoder_load = 0.7,95                                              output_type = types.RGB,96                                              axes = (0, 1))97    images = fn.decoders.image(jpegs,98                               device = device,99                               hw_decoder_load = 0.7,100                               output_type = types.RGB)101    images_sliced_2 = fn.slice(images,102                               anchor,103                               shape,104                               axes = (0, 1))105    return images_sliced_1, images_sliced_2106# TODO(januszl): check padding behavior107@pipeline_def108def create_decoder_crop_pipeline(data_path, device):109    jpegs, _ = fn.readers.file(file_root = data_path,110                               shard_id = 0,111                               num_shards = 1,112                               name = "Reader")113    crop_pos_x = fn.random.uniform(range=[0.1, 0.9])114    crop_pos_y = fn.random.uniform(range=[0.1, 0.9])115    w = 242116    h = 230117    images_crop_1 = fn.decoders.image_crop(jpegs,118                                           device = device,119                                           output_type = types.RGB,120                                           hw_decoder_load = 0.7,121                                           crop = (w, h),122                                           crop_pos_x = crop_pos_x,123                                           crop_pos_y = crop_pos_y)124    images = fn.decoders.image(jpegs,125                               device = device,126                               hw_decoder_load = 0.7,127                               output_type = types.RGB)128    images_crop_2 = fn.crop(images,129                            crop = (w, h),130                            crop_pos_x = crop_pos_x,131                            crop_pos_y = crop_pos_y)132    return images_crop_1, images_crop_2133@pipeline_def134def create_decoder_random_crop_pipeline(data_path, device):135    seed = 1234136    jpegs, _ = fn.readers.file(file_root = data_path,137                               shard_id = 0,138                               num_shards = 1,139                               name = "Reader")140    w = 242141    h = 230142    images_random_crop_1 = fn.decoders.image_random_crop(jpegs,143                                                         device = device,144                                                         output_type = types.RGB,145                                                         hw_decoder_load = 0.7,146                                                         seed = seed)147    images_random_crop_1 = fn.resize(images_random_crop_1, size = (w, h))148    images = fn.decoders.image(jpegs,149                               device = device,150                               hw_decoder_load = 0.7,151                               output_type = types.RGB)152    images_random_crop_2 = fn.random_resized_crop(images, size = (w, h), seed = seed)153    return images_random_crop_1, images_random_crop_2154def run_decode_fused(test_fun, path, img_type, batch, device, threads, validation_fun):155    data_path = os.path.join(test_data_root, path, img_type)156    pipe = test_fun(data_path=data_path, batch_size=batch, num_threads=threads, device_id=0, device=device, prefetch_queue_depth=1)157    pipe.build()158    iters = math.ceil(pipe.epoch_size("Reader") / batch)159    for _ in range(iters):160        out_1, out_2 = pipe.run()161        for img_1, img_2 in zip(out_1, out_2):162            img_1 = to_array(img_1)163            img_2 = to_array(img_2)164            assert validation_fun(img_1, img_2)165def test_image_decoder_fused():166    threads = 4167    batch_size = 10168    for test_fun in [create_decoder_slice_pipeline, create_decoder_crop_pipeline, create_decoder_random_crop_pipeline]:169        if test_fun == create_decoder_random_crop_pipeline:170            # random_resized_crop can properly handle border as it has pixels that are cropped out, while171            # plain resize folowing image_decoder_random_crop cannot do that and must duplicate the border pixels172            validation_fun = lambda x, y: np.mean(np.abs(x - y) < 0.5)173        else:174            validation_fun = lambda x, y: np.allclose(x, y)175        for device in {'cpu', 'mixed'}:176            for img_type in test_good_path:177                yield run_decode_fused, test_fun, good_path, img_type, batch_size, device, threads, validation_fun178def check_FastDCT_body(batch_size, img_type, device):179    data_path = os.path.join(test_data_root, good_path, img_type)180    compare_pipelines(decoder_pipe(data_path=data_path, batch_size=batch_size, num_threads=3,181                                   device_id=0, device=device, use_fast_idct=False),182                      decoder_pipe(data_path=data_path, batch_size=batch_size, num_threads=3,183                                   device_id=0, device='cpu', use_fast_idct=True),184                      # average difference should be no bigger by off-by-3185                      batch_size=batch_size, N_iterations=3, eps=3)186def test_FastDCT():187    for device in {'cpu', 'mixed'}:188        for batch_size in {1, 8}:189            for img_type in test_good_path:190              yield check_FastDCT_body, batch_size, img_type, device191def test_image_decoder_memory_stats():192    device = 'mixed'193    img_type = 'jpeg'194    def check(img_type, size, device, threads):195        data_path = os.path.join(test_data_root, good_path, img_type)196        # largest allocation should match our (in this case) memory padding settings197        # (assuming no reallocation was needed here as the hint is big enough)198        pattern = 'Device memory: \d+ allocations, largest = 16777216 bytes\n' + \199                  'Host \(pinned\) memory: \d+ allocations, largest = 8388608 bytes\n'200        with check_output_pattern(pattern):201            run_decode(data_path, size, device, threads, memory_stats=True)202    for size in {1, 10}:203        for threads in {1, random.choice([2, 3, 4])}:204            yield check, img_type, size, device, threads205batch_size_test = 16206@pipeline_def(batch_size=batch_size_test, device_id=0, num_threads=4)207def img_decoder_pipe(device, out_type, files):208    encoded, _ = fn.readers.file(files=files)209    decoded = fn.decoders.image(encoded, device=device, output_type=out_type)210    return decoded211def _testimpl_image_decoder_consistency(img_out_type, file_fmt, path, subdir='*', ext=None):212    eps = 1213    if file_fmt == 'jpeg' or file_fmt == 'mixed':214        eps = 4215    if ((file_fmt == 'jpeg2k' or file_fmt == 'mixed') and img_out_type == types.YCbCr):216        eps = 6217    files = get_img_files(os.path.join(test_data_root, path), subdir=subdir, ext=ext)218    compare_pipelines(img_decoder_pipe("cpu", out_type=img_out_type, files=files),219                      img_decoder_pipe("mixed", out_type=img_out_type, files=files),220                      batch_size=batch_size_test, N_iterations=3,221                      eps=eps)222def test_image_decoder_consistency():223    for out_img_type in [types.RGB, types.BGR, types.YCbCr, types.GRAY, types.ANY_DATA]:224        for file_fmt in test_good_path:225            path = os.path.join(good_path, file_fmt)226            yield _testimpl_image_decoder_consistency, out_img_type, file_fmt, path227        for file_fmt, path, ext in [("tiff", "db/single/multichannel/tiff_multichannel", 'tif'),228                                    ("jpeg2k", "db/single/multichannel/with_alpha", 'jp2'),229                                    ("png", "db/single/multichannel/with_alpha", 'png')]:230            subdir = None  # In those paths the images are not organized in subdirs231            yield _testimpl_image_decoder_consistency, out_img_type, file_fmt, path, subdir, ext232def _testimpl_image_decoder_tiff_with_alpha_16bit(device, out_type, path, ext):233    @pipeline_def(batch_size=1, device_id=0, num_threads=1)234    def pipe(device, out_type, files):235        encoded, _ = fn.readers.file(files=files)236        decoded = fn.decoders.image(encoded, device=device, output_type=out_type)237        peeked_shape = fn.peek_image_shape(encoded)238        return decoded, peeked_shape239    files = get_img_files(os.path.join(test_data_root, path), ext=ext, subdir=None)240    pipe = pipe(device, out_type=out_type, files=files)241    pipe.build()242    out, shape = pipe.run()243    if device == 'mixed':244        out = out.as_cpu()245    out = np.array(out[0])246    shape = np.array(shape[0])247    expected_channels = 4 if out_type == types.ANY_DATA else \248                        1 if out_type == types.GRAY else \249                        3250    assert out.shape[2] == expected_channels, \251        f"Expected {expected_channels} but got {out.shape[2]}"252def test_image_decoder_tiff_with_alpha_16bit():253    for device in ['cpu', 'mixed']:254        for out_type in [types.RGB, types.BGR, types.YCbCr, types.ANY_DATA]:255            path = "db/single/multichannel/with_alpha_16bit"256            for ext in [("png", "tiff", "jp2")]:257                yield _testimpl_image_decoder_tiff_with_alpha_16bit, device, out_type, path, ext258@pipeline_def(batch_size=batch_size_test, device_id=0, num_threads=4)259def decoder_pipe_with_name(decoder_op, file_root, device, use_fast_idct):260    encoded, _ = fn.readers.file(file_root=file_root)261    decoded = decoder_op(encoded, device=device, output_type=types.RGB, use_fast_idct=use_fast_idct,262                         seed=42)263    return decoded264def check_image_decoder_alias(new_op, old_op, file_root, device, use_fast_idct):265    new_pipe = decoder_pipe_with_name(new_op, file_root, device, use_fast_idct)266    legacy_pipe = decoder_pipe_with_name(old_op, file_root, device, use_fast_idct)267    compare_pipelines(new_pipe, legacy_pipe, batch_size=batch_size_test, N_iterations=3)268def test_image_decoder_alias():269    data_path = os.path.join(test_data_root, good_path, "jpeg")270    for new_op, old_op in [(fn.decoders.image, fn.image_decoder),271                           (fn.decoders.image_crop, fn.image_decoder_crop),272                           (fn.decoders.image_random_crop, fn.image_decoder_random_crop)]:273        for device in ["cpu", "mixed"]:274            for use_fast_idct in [True, False]:275                yield check_image_decoder_alias, new_op, old_op, data_path, device, use_fast_idct276@pipeline_def(batch_size=batch_size_test, device_id=0, num_threads=4)277def decoder_slice_pipe(decoder_op, file_root, device, use_fast_idct):278    encoded, _ = fn.readers.file(file_root=file_root)279    start = types.Constant(np.array([0., 0.]))280    end = types.Constant(np.array([0.5, 0.5]))281    decoded = decoder_op(encoded, start, end, device=device,282                         output_type=types.RGB, use_fast_idct=use_fast_idct)283    return decoded284def check_image_decoder_slice_alias(new_op, old_op, file_root, device, use_fast_idct):285    new_pipe = decoder_slice_pipe(new_op, file_root, device, use_fast_idct)286    legacy_pipe = decoder_slice_pipe(old_op, file_root, device, use_fast_idct)287    compare_pipelines(new_pipe, legacy_pipe, batch_size=batch_size_test, N_iterations=3)288def test_image_decoder_slice_alias():289    data_path = os.path.join(test_data_root, good_path, "jpeg")290    new_op, old_op = fn.decoders.image_slice, fn.image_decoder_slice291    for device in ["cpu", "mixed"]:292        for use_fast_idct in [True, False]:...

test_operator_decoder.py

Source:test_operator_decoder.py

1# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14from __future__ import print_function15from nvidia.dali.pipeline import Pipeline16import nvidia.dali.ops as ops17import nvidia.dali.types as types18import os19from test_utils import check_batch20from test_utils import compare_pipelines21from test_utils import RandomDataIterator22from test_utils import get_dali_extra_path23class DecoderPipeline(Pipeline):24    def __init__(self, data_path, batch_size, num_threads, device_id, device, use_fast_idct=False):25        super(DecoderPipeline, self).__init__(batch_size, num_threads, device_id, prefetch_queue_depth=1)26        self.input = ops.FileReader(file_root = data_path,27                                    shard_id = 0,28                                    num_shards = 1)29        self.decode = ops.ImageDecoder(device = device, output_type = types.RGB, use_fast_idct=use_fast_idct)30    def define_graph(self):31        inputs, labels = self.input(name="Reader")32        output = self.decode(inputs)33        return (output, labels)34test_data_root = get_dali_extra_path()35good_path = 'db/single'36missnamed_path = 'db/single/missnamed'37test_good_path = {'jpeg', 'mixed', 'png', 'tiff', 'pnm', 'bmp'}38test_missnamed_path = {'jpeg', 'png', 'tiff', 'pnm', 'bmp'}39def run_decode(data_path, batch, device, threads):40    pipe = DecoderPipeline(data_path=data_path, batch_size=batch, num_threads=threads, device_id=0, device=device)41    pipe.build()42    iters = pipe.epoch_size("Reader")43    for _ in range(iters):44        pipe.run()45def test_image_decoder():46    for device in {'cpu', 'mixed'}:47        for threads in {1, 2, 3, 4}:48            for size in {1, 10}:49                for img_type in test_good_path:50                    data_path = os.path.join(test_data_root, good_path, img_type)51                    run_decode(data_path, size, device, threads)52                    yield check, img_type, size, device, threads53def test_missnamed_host_decoder():54    for decoder in {'cpu', 'mixed'}:55        for threads in {1, 2, 3, 4}:56            for size in {1, 10}:57                for img_type in test_missnamed_path:58                    data_path = os.path.join(test_data_root, missnamed_path, img_type)59                    run_decode(data_path, size, decoder, threads)60                    yield check, img_type, size, decoder, threads61def check(img_type, size, device, threads):62    pass63class DecoderPipelineFastIDC(Pipeline):64    def __init__(self, data_path, batch_size, num_threads, use_fast_idct=False):65        super(DecoderPipelineFastIDC, self).__init__(batch_size, num_threads, 0, prefetch_queue_depth=1)66        self.input = ops.FileReader(file_root = data_path,67                                    shard_id = 0,68                                    num_shards = 1)69        self.decode = ops.ImageDecoder(device = 'cpu', output_type = types.RGB, use_fast_idct=use_fast_idct)70    def define_graph(self):71        inputs, labels = self.input(name="Reader")72        output = self.decode(inputs)73        return (output, labels)74def check_FastDCT_body(batch_size, img_type, device):75    data_path = os.path.join(test_data_root, good_path, img_type)76    compare_pipelines(DecoderPipeline(data_path=data_path, batch_size=batch_size, num_threads=3,77                                      device_id=0, device=device, use_fast_idct=False),78                      DecoderPipeline(data_path=data_path, batch_size=batch_size, num_threads=3,79                                      device_id=0, device='cpu', use_fast_idct=True),80                      # average difference should be no bigger by off-by-381                      batch_size=batch_size, N_iterations=3, eps=3)82def check_FastDCT():83    for device in {'cpu', 'mixed'}:84        for batch_size in {1, 8}:85            for img_type in test_good_path:...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.