How to use get_data_type method in autotest

Best Python code snippet using autotest_python

fileio.py

Source:fileio.py

...34    if hasattr(an_iterable, "ravel"):35        # It's a numpy array36        an_iterable = an_iterable.ravel().tolist()37    if an_iterable:38        data_type = get_data_type(an_iterable)39        data = encode_xdr(an_iterable, data_type)40    else:41        data = ""42    open(filename, "wb").write(data)43def get_data_type(something, default=None):44    """Given an iterable (list, numpy array, etc.) or a scalar (int, float, 45    complex, etc.), determines the type and returns a constants.DataTypes46    value representing the data type.47    Callers should be aware that passing an empty list will raise a TypeError48    because an empty list contains no clues as to the type of data it can49    contain. This weakness is specific to standard Python types. Numpy arrays 50    are immune from this problem because they specify a data type even when the51    array is empty.52    If a default is specified, this code will return that instead of raising53    an error when it can't determine the type.54    """55    data_type = constants.DataTypes.NONE56    if default:57        # In case I use the default, I map it to a DataTypes value58        default = constants.DataTypes.any_type_to_internal(default)59    if hasattr(something, "dtype"):60        # It's a numpy array or a numpy elemental value (e.g. a single float)61        data_type = constants.DataTypes.any_type_to_internal(str(something.dtype))62    elif util_misc.is_iterable(something, False):63        # It's not a numpy array but it is iterable so it's something like a 64        # list or tuple.65        if len(something):66            something = something[0]67            # It's unusual but possible to encounter a regular Python list 68            # containing numpy values, so here we check again for a numpy 69            # object.70            if hasattr(something, "dtype"):71                data_type = str(something.dtype)72            else:73                # It's a non-numpy elemental type like a Python int or float.74                data_type = type(something)75            data_type = constants.DataTypes.any_type_to_internal(data_type)76        else:77            # empty list78            if default:79                data_type = default80            else:81                raise TypeError("Can't determine the data type of an empty list")82    else:83        # It's a non-numpy elemental type like a Python int or float.84        data_type = constants.DataTypes.any_type_to_internal(type(something))85    if not data_type:86        if default:87            data_type = default88        else:89            raise ValueError("Unable to determine the data type")90    return data_type91###################       collapse/expand complexes       ###################92def collapse_complexes(data, conjugate_flag=False):93    """Given a list or other iterable that's a series of (real, imaginary)94    pairs, returns a list of complex numbers. For instance, given this list --95       [a, b, c, d, e, f]96    this function returns --97       [complex(a, b), complex(c, d), complex(e, f)]98    The returned list is a new list; the original is unchanged.99    """100    # This code was chosen for speed and efficiency. It creates an iterator101    # over the original list which gets called by izip. (izip() is the same102    # as the builtin zip() except that it returns elements one by one instead103    # of creating the whole list in memory.)104    # It's the fastest method of the 5 or 6 I tried, and I think it is also105    # very memory-efficient. 106    # I stole it from here:107    # http://stackoverflow.com/questions/4628290/pairs-from-single-list108    data_iter = iter(data)109    if not conjugate_flag:110        tmp = [complex(r, i) for r, i in zip(data_iter, data_iter)]111    else:112        tmp = [complex(r, -1*i) for r, i in zip(data_iter, data_iter)]113    return tmp114    115def expand_complexes(data):116    """Expands a list or tuple of complex numbers into a list of117    (real, imaginary) pairs. For instance, given this list of complex118    numbers --119       [za, zb, zc]120    this function returns --121       [za.real, za.imag, zb.real, zb.imag, zc.real, zc.imag]122    The returned list is a new list; the original is unchanged.123    124    You can also pass a numpy array as long as it is one-dimensional. 125    """126    # First I double the length of the list by adding empty elements to127    # the front.128    data = ([None] * len(data)) + list(data)129    # Now I overwrite the items in the list with item N being split130    # into real and imag and list[N/2] = real and list[N/2 + 1] = imag.131    j = 0132    for i in range(len(data) // 2, len(data)):133        data[j] = data[i].real134        j += 1135        data[j] = data[i].imag136        j += 1137    return data138#####################       decode/encode XDR       #####################139def decode_xdr(data, data_type, element_count):140    """Given a string of data in XDR format and a data type, returns141    an iterable (tuple or list) of Python objects representing the decoded142    data. data_type must be one of the constants from143    vespa.common.constants.DataTypes.ALL.144    145    element_count is the number of elements expected in the data.146    147    If the data is complex, the element count should be the number of complex148    numbers expected (despite the fact that XDR doesn't understand complex149    numbers).150    The companion function to this is encode_xdr().151    """152    p = xdrlib.Unpacker(data)153    if data_type in (constants.DataTypes.FLOAT32, constants.DataTypes.COMPLEX64):154        unpack_function = p.unpack_float155    elif data_type in (constants.DataTypes.FLOAT64, constants.DataTypes.COMPLEX128):156        unpack_function = p.unpack_double157    elif data_type in (constants.DataTypes.BYTE, constants.DataTypes.INT32):158        unpack_function = p.unpack_int159    elif data_type in (constants.DataTypes.BOOL, ):160        unpack_function = p.unpack_bool161    elif data_type in (constants.DataTypes.INT64,):162        unpack_function = p.unpack_hyper163    else:164        raise ValueError("Unknown data type '%s'" % data_type)165    if constants.DataTypes.is_complex(data_type):166        # XDR doesn't explicitly support complex numbers, so they're written167        # as pairs of floats (or doubles).168        element_count *= 2169    try:170        data = p.unpack_farray(element_count, unpack_function)171    except (xdrlib.Error, xdrlib.ConversionError) as instance:172        raise UnreadableDataError(instance.msg)173    # Calling p.done() here will raise an xdrlib.Error if unextracted174    # data remains (i.e. the code above is buggy or element_count is wrong)175    try:176        p.done()177    except xdrlib.Error:178        raise UnreadableDataError("More data in file than expected (XDR overrun)")179    if constants.DataTypes.is_complex(data_type):180        # Knit the (real, imaginary) pairs back together into complex numbers.181        data = collapse_complexes(data)182    return data183    184    185def encode_xdr(data, data_type):186    """Given a tuple or list of numbers (ints, floats, complex), returns187    an string representing the data in XDR format. You can also pass 188    single-dimension numpy arrays. Multi-dimension arrays raise an error.189    data_type must be one of the values in 190    vespa.common.constants.DataTypes.ALL.191    192    To reverse the encoding, use decode_xdr().193    """194    if constants.DataTypes.is_complex(data_type):195        # XDR doesn't understand complex numbers so I expand these into 196        # (real, imaginary) pairs.197        data = expand_complexes(data)198    p = xdrlib.Packer()199    # Decide which packing function to use200    if data_type in (constants.DataTypes.BOOL, ):201        function = p.pack_bool202    if data_type in (constants.DataTypes.BYTE, constants.DataTypes.INT32):203        function = p.pack_int204    if data_type in (constants.DataTypes.INT64, ):205        function = p.pack_hyper206    elif data_type in (constants.DataTypes.FLOAT32, constants.DataTypes.COMPLEX64):207        function = p.pack_float208    elif data_type in (constants.DataTypes.FLOAT64, constants.DataTypes.COMPLEX128):209        function = p.pack_double210    p.pack_farray(len(data), data, function)211    212    return p.get_buffer()213#####################       Unit tests       #####################214def _test_collapse_expand_complexes():215    import random216    import numpy217    random.seed()218    LIST_SIZE = random.randint(0, 1000)219    collapsed = [ ]220    raw = [ ]221    # Generate a bunch of random floats222    random_float = lambda: random.randint(-1000, 1000) + random.random()223    for i in range(LIST_SIZE):224        real = random_float()225        imaginary = random_float()226        raw.append(real)227        raw.append(imaginary)228        collapsed.append(complex(real, imaginary))229    assert(collapse_complexes(raw) == collapsed)230    assert(expand_complexes(collapsed) == raw)231    # Ensure the functions work with numpy arrays232    raw = numpy.array(raw)233    collapsed = numpy.array(collapsed)234    assert((numpy.array(collapse_complexes(raw)) == collapsed).all())235    assert((numpy.array(expand_complexes(collapsed)) == raw).all())236def _test_dump_iterable():237    import tempfile238    import os239    import numpy240    fd, filename = tempfile.mkstemp()241    os.close(fd)242    dump_iterable(filename, tuple(range(20)))       # tuple of ints243    dump_iterable(filename, list(range(20)))        # list of ints244    dump_iterable(filename, numpy.zeros(10))        # numpy 1D array of floats245    dump_iterable(filename, numpy.zeros( (5,5) ))   # numpy 2D array of floats246    os.remove(filename)247def _test_encode_decode_xdr():248    import random249    import numpy250    import vespa.common.util.math_ as util_math251    random.seed()252    LIST_SIZE = random.randint(0, 1000)253    # Generate a bunch of random floats254    random_float = lambda: random.randint(-1000, 1000) + random.random()255    original = [random_float() for i in range(LIST_SIZE)]256    s = encode_xdr(original, constants.DataTypes.FLOAT32)257    258    round_tripped = decode_xdr(s, constants.DataTypes.FLOAT32, LIST_SIZE)259        260    assert(all([util_math.eq(a, b) for (a, b) in zip(original, round_tripped)]))261    # Ensure the functions work with numpy arrays262    original = numpy.array(original)263    s = encode_xdr(original, constants.DataTypes.FLOAT32)264    265    round_tripped = decode_xdr(s, constants.DataTypes.FLOAT32, LIST_SIZE)266    267    round_tripped = numpy.array(round_tripped)268    269    assert(numpy.allclose(original, round_tripped))270def _test_get_data_type():271    # test ability to translate basic Python data types272    assert(get_data_type(True) == constants.DataTypes.BOOL)273    assert(get_data_type(42) == constants.DataTypes.INT32)274    assert(get_data_type(int(42)) == constants.DataTypes.INT64)275    assert(get_data_type(2.2) == constants.DataTypes.FLOAT64)276    assert(get_data_type(complex(1.1, 3.3)) == constants.DataTypes.COMPLEX128)277    # test ability to detect basic Python types inside iterables278    assert(get_data_type( (42, ) ) == constants.DataTypes.INT32)279    assert(get_data_type( [int(42), ] ) == constants.DataTypes.INT64)280    assert(get_data_type( (2.2, ) ) == constants.DataTypes.FLOAT64)281    assert(get_data_type( [complex(1.1, 3.3), ] ) == constants.DataTypes.COMPLEX128)282    # Test use of default283    assert(get_data_type([ ], float) == constants.DataTypes.FLOAT64)284    import numpy285    # Test numpy basic types286    assert(get_data_type(numpy.int8(42)) == constants.DataTypes.INT32)287    assert(get_data_type(numpy.int16(42)) == constants.DataTypes.INT32)288    assert(get_data_type(numpy.int32(42)) == constants.DataTypes.INT32)289    assert(get_data_type(numpy.int64(42)) == constants.DataTypes.INT64)290    assert(get_data_type(numpy.float32(2.2)) == constants.DataTypes.FLOAT32)291    assert(get_data_type(numpy.float64(2.2)) == constants.DataTypes.FLOAT64)292    assert(get_data_type(numpy.complex64(1+1j)) == constants.DataTypes.COMPLEX64)293    assert(get_data_type(numpy.complex128(1+1j)) == constants.DataTypes.COMPLEX128)294    # Test lists of numpy basic types295    assert(get_data_type( [numpy.int8(42)] ) == constants.DataTypes.INT32)296    assert(get_data_type( [numpy.int16(42)] ) == constants.DataTypes.INT32)297    assert(get_data_type( [numpy.int32(42)] ) == constants.DataTypes.INT32)298    assert(get_data_type( [numpy.int64(42)] ) == constants.DataTypes.INT64)299    assert(get_data_type( [numpy.float32(2.2)] ) == constants.DataTypes.FLOAT32)300    assert(get_data_type( [numpy.float64(2.2)] ) == constants.DataTypes.FLOAT64)301    assert(get_data_type( [numpy.complex64(1+1j)] ) == constants.DataTypes.COMPLEX64)302    assert(get_data_type( [numpy.complex128(1+1j)] ) == constants.DataTypes.COMPLEX128)303    # Test numpy arrays304    assert(get_data_type(numpy.zeros(5, numpy.int8)) == constants.DataTypes.INT32)305    assert(get_data_type(numpy.zeros(5, numpy.int16)) == constants.DataTypes.INT32)306    assert(get_data_type(numpy.zeros(5, numpy.int32)) == constants.DataTypes.INT32)307    assert(get_data_type(numpy.zeros(5, numpy.int64)) == constants.DataTypes.INT64)308    assert(get_data_type(numpy.zeros(5, numpy.float32)) == constants.DataTypes.FLOAT32)309    assert(get_data_type(numpy.zeros(5, numpy.float64)) == constants.DataTypes.FLOAT64)310    assert(get_data_type(numpy.zeros(5, numpy.complex64)) == constants.DataTypes.COMPLEX64)311    assert(get_data_type(numpy.zeros(5, numpy.complex128)) == constants.DataTypes.COMPLEX128)312if __name__ == '__main__':313    _test_get_data_type()314    _test_collapse_expand_complexes()315    _test_encode_decode_xdr()...

utils_test.py

Source:utils_test.py

...42    assert sanitize_dynamic_axes(axes) == \43            tuple(reversed(C.Axis.default_input_variable_dynamic_axes()))44    assert (C.Axis.default_dynamic_axis(),) == \45            sanitize_dynamic_axes(C.Axis.default_dynamic_axis())46def test_get_data_type():47    pa32 = C.parameter(init=np.asarray(2, dtype=np.float32))48    pa64 = C.parameter(init=np.asarray(2, dtype=np.float64))49    pl = C.placeholder(shape=(2))50    c = C.constant(value=3.0)51    n32 = AA(1, dtype=np.float32)52    n64 = AA(1, dtype=np.float64)53    assert get_data_type(pa32) == np.float3254    assert get_data_type(pa32, n32) == np.float3255    assert get_data_type(n32, n32) == np.float3256    assert get_data_type(n32, n64) == np.float6457    assert get_data_type(pl, n64) == np.float6458    assert get_data_type(pl, n32) == np.float3259    assert get_data_type(pl, pl) is None60    # variable's type shall take precedence over provided data61    assert get_data_type(pa32, n64) == np.float3262    assert get_data_type(pa64, n64) == np.float6463    assert get_data_type(pa32, pl, n64) == np.float3264    assert get_data_type(pa64, pl, n64) == np.float6465    66    assert get_data_type(np.float64(1)) == np.float6467    assert get_data_type(np.float32(1)) == np.float3268    assert get_data_type(np.int64(1)) == np.float32  # special case for cntk69    assert get_data_type(1) == np.float3270    assert get_data_type(1.0) == np.float3271def test_sanitize_batch_sparse():72    batch = [csr([[1,0,2],[2,3,0]]),73             csr([5,0,1])]74    var = sequence.input_variable(3, is_sparse=True)75    b = sanitize_batch(var, batch)76    # 2 sequences, with max seq len of 2 and dimension 377    assert b.shape == (2,2,3)78@pytest.mark.parametrize("batch, seq_starts, expected", [79    ([AA([5, 6, 7]), AA([8])],80       [True, False],81       [[2, 1, 1], [1, 0, 0]]),82    ([AA([5]), AA([8])],83       [True, False],84       [[2], [1]]),...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.