How to use member method in hypothesis
Best Python code snippet using hypothesis
enum.py
Source:enum.py 
1import sys2from types import MappingProxyType, DynamicClassAttribute3# try _collections first to reduce startup cost4try:5 from _collections import OrderedDict6except ImportError:7 from collections import OrderedDict8__all__ = [9 'EnumMeta',10 'Enum', 'IntEnum', 'Flag', 'IntFlag',11 'auto', 'unique',12 ]13def _is_descriptor(obj):14 """Returns True if obj is a descriptor, False otherwise."""15 return (16 hasattr(obj, '__get__') or17 hasattr(obj, '__set__') or18 hasattr(obj, '__delete__'))19def _is_dunder(name):20 """Returns True if a __dunder__ name, False otherwise."""21 return (len(name) > 4 and22 name[:2] == name[-2:] == '__' and23 name[2] != '_' and24 name[-3] != '_')25def _is_sunder(name):26 """Returns True if a _sunder_ name, False otherwise."""27 return (len(name) > 2 and28 name[0] == name[-1] == '_' and29 name[1:2] != '_' and30 name[-2:-1] != '_')31def _make_class_unpicklable(cls):32 """Make the given class un-picklable."""33 def _break_on_call_reduce(self, proto):34 raise TypeError('%r cannot be pickled' % self)35 cls.__reduce_ex__ = _break_on_call_reduce36 cls.__module__ = '<unknown>'37_auto_null = object()38class auto:39 """40 Instances are replaced with an appropriate value in Enum class suites.41 """42 value = _auto_null43class _EnumDict(dict):44 """Track enum member order and ensure member names are not reused.45 EnumMeta will use the names found in self._member_names as the46 enumeration member names.47 """48 def __init__(self):49 super().__init__()50 self._member_names = []51 self._last_values = []52 self._ignore = []53 def __setitem__(self, key, value):54 """Changes anything not dundered or not a descriptor.55 If an enum member name is used twice, an error is raised; duplicate56 values are not checked for.57 Single underscore (sunder) names are reserved.58 """59 if _is_sunder(key):60 if key not in (61 '_order_', '_create_pseudo_member_',62 '_generate_next_value_', '_missing_', '_ignore_',63 ):64 raise ValueError('_names_ are reserved for future Enum use')65 if key == '_generate_next_value_':66 setattr(self, '_generate_next_value', value)67 elif key == '_ignore_':68 if isinstance(value, str):69 value = value.replace(',',' ').split()70 else:71 value = list(value)72 self._ignore = value73 already = set(value) & set(self._member_names)74 if already:75 raise ValueError('_ignore_ cannot specify already set names: %r' % (already, ))76 elif _is_dunder(key):77 if key == '__order__':78 key = '_order_'79 elif key in self._member_names:80 # descriptor overwriting an enum?81 raise TypeError('Attempted to reuse key: %r' % key)82 elif key in self._ignore:83 pass84 elif not _is_descriptor(value):85 if key in self:86 # enum overwriting a descriptor?87 raise TypeError('%r already defined as: %r' % (key, self[key]))88 if isinstance(value, auto):89 if value.value == _auto_null:90 value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:])91 value = value.value92 self._member_names.append(key)93 self._last_values.append(value)94 super().__setitem__(key, value)95# Dummy value for Enum as EnumMeta explicitly checks for it, but of course96# until EnumMeta finishes running the first time the Enum class doesn't exist.97# This is also why there are checks in EnumMeta like `if Enum is not None`98Enum = None99class EnumMeta(type):100 """Metaclass for Enum"""101 @classmethod102 def __prepare__(metacls, cls, bases):103 # create the namespace dict104 enum_dict = _EnumDict()105 # inherit previous flags and _generate_next_value_ function106 member_type, first_enum = metacls._get_mixins_(bases)107 if first_enum is not None:108 enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None)109 return enum_dict110 def __new__(metacls, cls, bases, classdict):111 # an Enum class is final once enumeration items have been defined; it112 # cannot be mixed with other types (int, float, etc.) if it has an113 # inherited __new__ unless a new __new__ is defined (or the resulting114 # class will fail).115 #116 # remove any keys listed in _ignore_117 classdict.setdefault('_ignore_', []).append('_ignore_')118 ignore = classdict['_ignore_']119 for key in ignore:120 classdict.pop(key, None)121 member_type, first_enum = metacls._get_mixins_(bases)122 __new__, save_new, use_args = metacls._find_new_(classdict, member_type,123 first_enum)124 # save enum items into separate mapping so they don't get baked into125 # the new class126 enum_members = {k: classdict[k] for k in classdict._member_names}127 for name in classdict._member_names:128 del classdict[name]129 # adjust the sunders130 _order_ = classdict.pop('_order_', None)131 # check for illegal enum names (any others?)132 invalid_names = set(enum_members) & {'mro', ''}133 if invalid_names:134 raise ValueError('Invalid enum member name: {0}'.format(135 ','.join(invalid_names)))136 # create a default docstring if one has not been provided137 if '__doc__' not in classdict:138 classdict['__doc__'] = 'An enumeration.'139 # create our new Enum type140 enum_class = super().__new__(metacls, cls, bases, classdict)141 enum_class._member_names_ = [] # names in definition order142 enum_class._member_map_ = OrderedDict() # name->value map143 enum_class._member_type_ = member_type144 # save DynamicClassAttribute attributes from super classes so we know145 # if we can take the shortcut of storing members in the class dict146 dynamic_attributes = {k for c in enum_class.mro()147 for k, v in c.__dict__.items()148 if isinstance(v, DynamicClassAttribute)}149 # Reverse value->name map for hashable values.150 enum_class._value2member_map_ = {}151 # If a custom type is mixed into the Enum, and it does not know how152 # to pickle itself, pickle.dumps will succeed but pickle.loads will153 # fail. Rather than have the error show up later and possibly far154 # from the source, sabotage the pickle protocol for this class so155 # that pickle.dumps also fails.156 #157 # However, if the new class implements its own __reduce_ex__, do not158 # sabotage -- it's on them to make sure it works correctly. We use159 # __reduce_ex__ instead of any of the others as it is preferred by160 # pickle over __reduce__, and it handles all pickle protocols.161 if '__reduce_ex__' not in classdict:162 if member_type is not object:163 methods = ('__getnewargs_ex__', '__getnewargs__',164 '__reduce_ex__', '__reduce__')165 if not any(m in member_type.__dict__ for m in methods):166 _make_class_unpicklable(enum_class)167 # instantiate them, checking for duplicates as we go168 # we instantiate first instead of checking for duplicates first in case169 # a custom __new__ is doing something funky with the values -- such as170 # auto-numbering ;)171 for member_name in classdict._member_names:172 value = enum_members[member_name]173 if not isinstance(value, tuple):174 args = (value, )175 else:176 args = value177 if member_type is tuple: # special case for tuple enums178 args = (args, ) # wrap it one more time179 if not use_args:180 enum_member = __new__(enum_class)181 if not hasattr(enum_member, '_value_'):182 enum_member._value_ = value183 else:184 enum_member = __new__(enum_class, *args)185 if not hasattr(enum_member, '_value_'):186 if member_type is object:187 enum_member._value_ = value188 else:189 enum_member._value_ = member_type(*args)190 value = enum_member._value_191 enum_member._name_ = member_name192 enum_member.__objclass__ = enum_class193 enum_member.__init__(*args)194 # If another member with the same value was already defined, the195 # new member becomes an alias to the existing one.196 for name, canonical_member in enum_class._member_map_.items():197 if canonical_member._value_ == enum_member._value_:198 enum_member = canonical_member199 break200 else:201 # Aliases don't appear in member names (only in __members__).202 enum_class._member_names_.append(member_name)203 # performance boost for any member that would not shadow204 # a DynamicClassAttribute205 if member_name not in dynamic_attributes:206 setattr(enum_class, member_name, enum_member)207 # now add to _member_map_208 enum_class._member_map_[member_name] = enum_member209 try:210 # This may fail if value is not hashable. We can't add the value211 # to the map, and by-value lookups for this value will be212 # linear.213 enum_class._value2member_map_[value] = enum_member214 except TypeError:215 pass216 # double check that repr and friends are not the mixin's or various217 # things break (such as pickle)218 for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'):219 class_method = getattr(enum_class, name)220 obj_method = getattr(member_type, name, None)221 enum_method = getattr(first_enum, name, None)222 if obj_method is not None and obj_method is class_method:223 setattr(enum_class, name, enum_method)224 # replace any other __new__ with our own (as long as Enum is not None,225 # anyway) -- again, this is to support pickle226 if Enum is not None:227 # if the user defined their own __new__, save it before it gets228 # clobbered in case they subclass later229 if save_new:230 enum_class.__new_member__ = __new__231 enum_class.__new__ = Enum.__new__232 # py3 support for definition order (helps keep py2/py3 code in sync)233 if _order_ is not None:234 if isinstance(_order_, str):235 _order_ = _order_.replace(',', ' ').split()236 if _order_ != enum_class._member_names_:237 raise TypeError('member order does not match _order_')238 return enum_class239 def __bool__(self):240 """241 classes/types should always be True.242 """243 return True244 def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1):245 """Either returns an existing member, or creates a new enum class.246 This method is used both when an enum class is given a value to match247 to an enumeration member (i.e. Color(3)) and for the functional API248 (i.e. Color = Enum('Color', names='RED GREEN BLUE')).249 When used for the functional API:250 `value` will be the name of the new class.251 `names` should be either a string of white-space/comma delimited names252 (values will start at `start`), or an iterator/mapping of name, value pairs.253 `module` should be set to the module this class is being created in;254 if it is not set, an attempt to find that module will be made, but if255 it fails the class will not be picklable.256 `qualname` should be set to the actual location this class can be found257 at in its module; by default it is set to the global scope. If this is258 not correct, unpickling will fail in some circumstances.259 `type`, if set, will be mixed in as the first base class.260 """261 if names is None: # simple value lookup262 return cls.__new__(cls, value)263 # otherwise, functional API: we're creating a new Enum type264 return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start)265 def __contains__(cls, member):266 if not isinstance(member, Enum):267 import warnings268 warnings.warn(269 "using non-Enums in containment checks will raise "270 "TypeError in Python 3.8",271 DeprecationWarning, 2)272 return isinstance(member, cls) and member._name_ in cls._member_map_273 def __delattr__(cls, attr):274 # nicer error message when someone tries to delete an attribute275 # (see issue19025).276 if attr in cls._member_map_:277 raise AttributeError(278 "%s: cannot delete Enum member." % cls.__name__)279 super().__delattr__(attr)280 def __dir__(self):281 return (['__class__', '__doc__', '__members__', '__module__'] +282 self._member_names_)283 def __getattr__(cls, name):284 """Return the enum member matching `name`285 We use __getattr__ instead of descriptors or inserting into the enum286 class' __dict__ in order to support `name` and `value` being both287 properties for enum members (which live in the class' __dict__) and288 enum members themselves.289 """290 if _is_dunder(name):291 raise AttributeError(name)292 try:293 return cls._member_map_[name]294 except KeyError:295 raise AttributeError(name) from None296 def __getitem__(cls, name):297 return cls._member_map_[name]298 def __iter__(cls):299 return (cls._member_map_[name] for name in cls._member_names_)300 def __len__(cls):301 return len(cls._member_names_)302 @property303 def __members__(cls):304 """Returns a mapping of member name->value.305 This mapping lists all enum members, including aliases. Note that this306 is a read-only view of the internal mapping.307 """308 return MappingProxyType(cls._member_map_)309 def __repr__(cls):310 return "<enum %r>" % cls.__name__311 def __reversed__(cls):312 return (cls._member_map_[name] for name in reversed(cls._member_names_))313 def __setattr__(cls, name, value):314 """Block attempts to reassign Enum members.315 A simple assignment to the class namespace only changes one of the316 several possible ways to get an Enum member from the Enum class,317 resulting in an inconsistent Enumeration.318 """319 member_map = cls.__dict__.get('_member_map_', {})320 if name in member_map:321 raise AttributeError('Cannot reassign members.')322 super().__setattr__(name, value)323 def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1):324 """Convenience method to create a new Enum class.325 `names` can be:326 * A string containing member names, separated either with spaces or327 commas. Values are incremented by 1 from `start`.328 * An iterable of member names. Values are incremented by 1 from `start`.329 * An iterable of (member name, value) pairs.330 * A mapping of member name -> value pairs.331 """332 metacls = cls.__class__333 bases = (cls, ) if type is None else (type, cls)334 _, first_enum = cls._get_mixins_(bases)335 classdict = metacls.__prepare__(class_name, bases)336 # special processing needed for names?337 if isinstance(names, str):338 names = names.replace(',', ' ').split()339 if isinstance(names, (tuple, list)) and names and isinstance(names[0], str):340 original_names, names = names, []341 last_values = []342 for count, name in enumerate(original_names):343 value = first_enum._generate_next_value_(name, start, count, last_values[:])344 last_values.append(value)345 names.append((name, value))346 # Here, names is either an iterable of (name, value) or a mapping.347 for item in names:348 if isinstance(item, str):349 member_name, member_value = item, names[item]350 else:351 member_name, member_value = item352 classdict[member_name] = member_value353 enum_class = metacls.__new__(metacls, class_name, bases, classdict)354 # TODO: replace the frame hack if a blessed way to know the calling355 # module is ever developed356 if module is None:357 try:358 module = sys._getframe(2).f_globals['__name__']359 except (AttributeError, ValueError, KeyError) as exc:360 pass361 if module is None:362 _make_class_unpicklable(enum_class)363 else:364 enum_class.__module__ = module365 if qualname is not None:366 enum_class.__qualname__ = qualname367 return enum_class368 @staticmethod369 def _get_mixins_(bases):370 """Returns the type for creating enum members, and the first inherited371 enum class.372 bases: the tuple of bases that was given to __new__373 """374 if not bases:375 return object, Enum376 def _find_data_type(bases):377 for chain in bases:378 for base in chain.__mro__:379 if base is object:380 continue381 elif '__new__' in base.__dict__:382 if issubclass(base, Enum):383 continue384 return base385 # ensure final parent class is an Enum derivative, find any concrete386 # data type, and check that Enum has no members387 first_enum = bases[-1]388 if not issubclass(first_enum, Enum):389 raise TypeError("new enumerations should be created as "390 "`EnumName([mixin_type, ...] [data_type,] enum_type)`")391 member_type = _find_data_type(bases) or object392 if first_enum._member_names_:393 raise TypeError("Cannot extend enumerations")394 return member_type, first_enum395 @staticmethod396 def _find_new_(classdict, member_type, first_enum):397 """Returns the __new__ to be used for creating the enum members.398 classdict: the class dictionary given to __new__399 member_type: the data type whose __new__ will be used by default400 first_enum: enumeration to check for an overriding __new__401 """402 # now find the correct __new__, checking to see of one was defined403 # by the user; also check earlier enum classes in case a __new__ was404 # saved as __new_member__405 __new__ = classdict.get('__new__', None)406 # should __new__ be saved as __new_member__ later?407 save_new = __new__ is not None408 if __new__ is None:409 # check all possibles for __new_member__ before falling back to410 # __new__411 for method in ('__new_member__', '__new__'):412 for possible in (member_type, first_enum):413 target = getattr(possible, method, None)414 if target not in {415 None,416 None.__new__,417 object.__new__,418 Enum.__new__,419 }:420 __new__ = target421 break422 if __new__ is not None:423 break424 else:425 __new__ = object.__new__426 # if a non-object.__new__ is used then whatever value/tuple was427 # assigned to the enum member name will be passed to __new__ and to the428 # new enum member's __init__429 if __new__ is object.__new__:430 use_args = False431 else:432 use_args = True433 return __new__, save_new, use_args434class Enum(metaclass=EnumMeta):435 """Generic enumeration.436 Derive from this class to define new enumerations.437 """438 def __new__(cls, value):439 # all enum instances are actually created during class construction440 # without calling this method; this method is called by the metaclass'441 # __call__ (i.e. Color(3) ), and by pickle442 if type(value) is cls:443 # For lookups like Color(Color.RED)444 return value445 # by-value search for a matching enum member446 # see if it's in the reverse mapping (for hashable values)447 try:448 return cls._value2member_map_[value]449 except KeyError:450 # Not found, no need to do long O(n) search451 pass452 except TypeError:453 # not there, now do long search -- O(n) behavior454 for member in cls._member_map_.values():455 if member._value_ == value:456 return member457 # still not found -- try _missing_ hook458 try:459 exc = None460 result = cls._missing_(value)461 except Exception as e:462 exc = e463 result = None464 if isinstance(result, cls):465 return result466 else:467 ve_exc = ValueError("%r is not a valid %s" % (value, cls.__name__))468 if result is None and exc is None:469 raise ve_exc470 elif exc is None:471 exc = TypeError(472 'error in %s._missing_: returned %r instead of None or a valid member'473 % (cls.__name__, result)474 )475 exc.__context__ = ve_exc476 raise exc477 def _generate_next_value_(name, start, count, last_values):478 for last_value in reversed(last_values):479 try:480 return last_value + 1481 except TypeError:482 pass483 else:484 return start485 @classmethod486 def _missing_(cls, value):487 raise ValueError("%r is not a valid %s" % (value, cls.__name__))488 def __repr__(self):489 return "<%s.%s: %r>" % (490 self.__class__.__name__, self._name_, self._value_)491 def __str__(self):492 return "%s.%s" % (self.__class__.__name__, self._name_)493 def __dir__(self):494 added_behavior = [495 m496 for cls in self.__class__.mro()497 for m in cls.__dict__498 if m[0] != '_' and m not in self._member_map_499 ]500 return (['__class__', '__doc__', '__module__'] + added_behavior)501 def __format__(self, format_spec):502 # mixed-in Enums should use the mixed-in type's __format__, otherwise503 # we can get strange results with the Enum name showing up instead of504 # the value505 # pure Enum branch506 if self._member_type_ is object:507 cls = str508 val = str(self)509 # mix-in branch510 else:511 cls = self._member_type_512 val = self._value_513 return cls.__format__(val, format_spec)514 def __hash__(self):515 return hash(self._name_)516 def __reduce_ex__(self, proto):517 return self.__class__, (self._value_, )518 # DynamicClassAttribute is used to provide access to the `name` and519 # `value` properties of enum members while keeping some measure of520 # protection from modification, while still allowing for an enumeration521 # to have members named `name` and `value`. This works because enumeration522 # members are not set directly on the enum class -- __getattr__ is523 # used to look them up.524 @DynamicClassAttribute525 def name(self):526 """The name of the Enum member."""527 return self._name_528 @DynamicClassAttribute529 def value(self):530 """The value of the Enum member."""531 return self._value_532 @classmethod533 def _convert(cls, name, module, filter, source=None):534 """535 Create a new Enum subclass that replaces a collection of global constants536 """537 # convert all constants from source (or module) that pass filter() to538 # a new Enum called name, and export the enum and its members back to539 # module;540 # also, replace the __reduce_ex__ method so unpickling works in541 # previous Python versions542 module_globals = vars(sys.modules[module])543 if source:544 source = vars(source)545 else:546 source = module_globals547 # We use an OrderedDict of sorted source keys so that the548 # _value2member_map is populated in the same order every time549 # for a consistent reverse mapping of number to name when there550 # are multiple names for the same number rather than varying551 # between runs due to hash randomization of the module dictionary.552 members = [553 (name, source[name])554 for name in source.keys()555 if filter(name)]556 try:557 # sort by value558 members.sort(key=lambda t: (t[1], t[0]))559 except TypeError:560 # unless some values aren't comparable, in which case sort by name561 members.sort(key=lambda t: t[0])562 cls = cls(name, members, module=module)563 cls.__reduce_ex__ = _reduce_ex_by_name564 module_globals.update(cls.__members__)565 module_globals[name] = cls566 return cls567class IntEnum(int, Enum):568 """Enum where members are also (and must be) ints"""569def _reduce_ex_by_name(self, proto):570 return self.name571class Flag(Enum):572 """Support for flags"""573 def _generate_next_value_(name, start, count, last_values):574 """575 Generate the next value when not given.576 name: the name of the member577 start: the initital start value or None578 count: the number of existing members579 last_value: the last value assigned or None580 """581 if not count:582 return start if start is not None else 1583 for last_value in reversed(last_values):584 try:585 high_bit = _high_bit(last_value)586 break587 except Exception:588 raise TypeError('Invalid Flag value: %r' % last_value) from None589 return 2 ** (high_bit+1)590 @classmethod591 def _missing_(cls, value):592 original_value = value593 if value < 0:594 value = ~value595 possible_member = cls._create_pseudo_member_(value)596 if original_value < 0:597 possible_member = ~possible_member598 return possible_member599 @classmethod600 def _create_pseudo_member_(cls, value):601 """602 Create a composite member iff value contains only members.603 """604 pseudo_member = cls._value2member_map_.get(value, None)605 if pseudo_member is None:606 # verify all bits are accounted for607 _, extra_flags = _decompose(cls, value)608 if extra_flags:609 raise ValueError("%r is not a valid %s" % (value, cls.__name__))610 # construct a singleton enum pseudo-member611 pseudo_member = object.__new__(cls)612 pseudo_member._name_ = None613 pseudo_member._value_ = value614 # use setdefault in case another thread already created a composite615 # with this value616 pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)617 return pseudo_member618 def __contains__(self, other):619 if not isinstance(other, self.__class__):620 import warnings621 warnings.warn(622 "using non-Flags in containment checks will raise "623 "TypeError in Python 3.8",624 DeprecationWarning, 2)625 return False626 return other._value_ & self._value_ == other._value_627 def __repr__(self):628 cls = self.__class__629 if self._name_ is not None:630 return '<%s.%s: %r>' % (cls.__name__, self._name_, self._value_)631 members, uncovered = _decompose(cls, self._value_)632 return '<%s.%s: %r>' % (633 cls.__name__,634 '|'.join([str(m._name_ or m._value_) for m in members]),635 self._value_,636 )637 def __str__(self):638 cls = self.__class__639 if self._name_ is not None:640 return '%s.%s' % (cls.__name__, self._name_)641 members, uncovered = _decompose(cls, self._value_)642 if len(members) == 1 and members[0]._name_ is None:643 return '%s.%r' % (cls.__name__, members[0]._value_)644 else:645 return '%s.%s' % (646 cls.__name__,647 '|'.join([str(m._name_ or m._value_) for m in members]),648 )649 def __bool__(self):650 return bool(self._value_)651 def __or__(self, other):652 if not isinstance(other, self.__class__):653 return NotImplemented654 return self.__class__(self._value_ | other._value_)655 def __and__(self, other):656 if not isinstance(other, self.__class__):657 return NotImplemented658 return self.__class__(self._value_ & other._value_)659 def __xor__(self, other):660 if not isinstance(other, self.__class__):661 return NotImplemented662 return self.__class__(self._value_ ^ other._value_)663 def __invert__(self):664 members, uncovered = _decompose(self.__class__, self._value_)665 inverted = self.__class__(0)666 for m in self.__class__:667 if m not in members and not (m._value_ & self._value_):668 inverted = inverted | m669 return self.__class__(inverted)670class IntFlag(int, Flag):671 """Support for integer-based Flags"""672 @classmethod673 def _missing_(cls, value):674 if not isinstance(value, int):675 raise ValueError("%r is not a valid %s" % (value, cls.__name__))676 new_member = cls._create_pseudo_member_(value)677 return new_member678 @classmethod679 def _create_pseudo_member_(cls, value):680 pseudo_member = cls._value2member_map_.get(value, None)681 if pseudo_member is None:682 need_to_create = [value]683 # get unaccounted for bits684 _, extra_flags = _decompose(cls, value)685 # timer = 10686 while extra_flags:687 # timer -= 1688 bit = _high_bit(extra_flags)689 flag_value = 2 ** bit690 if (flag_value not in cls._value2member_map_ and691 flag_value not in need_to_create692 ):693 need_to_create.append(flag_value)694 if extra_flags == -flag_value:695 extra_flags = 0696 else:697 extra_flags ^= flag_value698 for value in reversed(need_to_create):699 # construct singleton pseudo-members700 pseudo_member = int.__new__(cls, value)701 pseudo_member._name_ = None702 pseudo_member._value_ = value703 # use setdefault in case another thread already created a composite704 # with this value705 pseudo_member = cls._value2member_map_.setdefault(value, pseudo_member)706 return pseudo_member707 def __or__(self, other):708 if not isinstance(other, (self.__class__, int)):709 return NotImplemented710 result = self.__class__(self._value_ | self.__class__(other)._value_)711 return result712 def __and__(self, other):713 if not isinstance(other, (self.__class__, int)):714 return NotImplemented715 return self.__class__(self._value_ & self.__class__(other)._value_)716 def __xor__(self, other):717 if not isinstance(other, (self.__class__, int)):718 return NotImplemented719 return self.__class__(self._value_ ^ self.__class__(other)._value_)720 __ror__ = __or__721 __rand__ = __and__722 __rxor__ = __xor__723 def __invert__(self):724 result = self.__class__(~self._value_)725 return result726def _high_bit(value):727 """returns index of highest bit, or -1 if value is zero or negative"""728 return value.bit_length() - 1729def unique(enumeration):730 """Class decorator for enumerations ensuring unique member values."""731 duplicates = []732 for name, member in enumeration.__members__.items():733 if name != member.name:734 duplicates.append((name, member.name))735 if duplicates:736 alias_details = ', '.join(737 ["%s -> %s" % (alias, name) for (alias, name) in duplicates])738 raise ValueError('duplicate values found in %r: %s' %739 (enumeration, alias_details))740 return enumeration741def _decompose(flag, value):742 """Extract all members from the value."""743 # _decompose is only called if the value is not named744 not_covered = value745 negative = value < 0746 # issue29167: wrap accesses to _value2member_map_ in a list to avoid race747 # conditions between iterating over it and having more pseudo-748 # members added to it749 if negative:750 # only check for named flags751 flags_to_check = [752 (m, v)753 for v, m in list(flag._value2member_map_.items())754 if m.name is not None755 ]756 else:757 # check for named flags and powers-of-two flags758 flags_to_check = [759 (m, v)760 for v, m in list(flag._value2member_map_.items())761 if m.name is not None or _power_of_two(v)762 ]763 members = []764 for member, member_value in flags_to_check:765 if member_value and member_value & value == member_value:766 members.append(member)767 not_covered &= ~member_value768 if not members and value in flag._value2member_map_:769 members.append(flag._value2member_map_[value])770 members.sort(key=lambda m: m._value_, reverse=True)771 if len(members) > 1 and members[0].value == value:772 # we have the breakdown, don't need the value member itself773 members.pop(0)774 return members, not_covered775def _power_of_two(value):776 if value < 1:777 return False...
apache2_mod_proxy.py
Source:apache2_mod_proxy.py
1#!/usr/bin/python2# -*- coding: utf-8 -*-3# (c) 2016, Olivier Boukili <boukili.olivier@gmail.com>4#5# This file is part of Ansible.6#7# Ansible is free software: you can redistribute it and/or modify8# it under the terms of the GNU General Public License as published by9# the Free Software Foundation, either version 3 of the License, or10# (at your option) any later version.11#12# Ansible is distributed in the hope that it will be useful,13# but WITHOUT ANY WARRANTY; without even the implied warranty of14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the15# GNU General Public License for more details.16#17# You should have received a copy of the GNU General Public License18# along with Ansible. If not, see <http://www.gnu.org/licenses/>.19ANSIBLE_METADATA = {'metadata_version': '1.0',20 'status': ['preview'],21 'supported_by': 'community'}22DOCUMENTATION = '''23---24module: apache2_mod_proxy25author: Olivier Boukili (@oboukili)"26version_added: "2.2"27short_description: Set and/or get members' attributes of an Apache httpd 2.4 mod_proxy balancer pool28description:29 - Set and/or get members' attributes of an Apache httpd 2.4 mod_proxy balancer30 pool, using HTTP POST and GET requests. The httpd mod_proxy balancer-member31 status page has to be enabled and accessible, as this module relies on parsing32 this page. This module supports ansible check_mode, and requires BeautifulSoup33 python module.34options:35 balancer_url_suffix:36 default: /balancer-manager/37 description:38 - Suffix of the balancer pool url required to access the balancer pool39 status page (e.g. balancer_vhost[:port]/balancer_url_suffix).40 required: false41 balancer_vhost:42 default: None43 description:44 - (ipv4|ipv6|fqdn):port of the Apache httpd 2.4 mod_proxy balancer pool.45 required: true46 member_host:47 default: None48 description:49 - (ipv4|ipv6|fqdn) of the balancer member to get or to set attributes to.50 Port number is autodetected and should not be specified here.51 If undefined, apache2_mod_proxy module will return a members list of52 dictionaries of all the current balancer pool members' attributes.53 required: false54 state:55 default: None56 description:57 - Desired state of the member host.58 (absent|disabled),drained,hot_standby,ignore_errors can be59 simultaneously invoked by separating them with a comma (e.g. state=drained,ignore_errors).60 required: false61 choices: ["present", "absent", "enabled", "disabled", "drained", "hot_standby", "ignore_errors"]62 tls:63 default: false64 description:65 - Use https to access balancer management page.66 choices: ["true", "false"]67 validate_certs:68 default: true69 description:70 - Validate ssl/tls certificates.71 choices: ["true", "false"]72'''73EXAMPLES = '''74# Get all current balancer pool members' attributes:75- apache2_mod_proxy:76 balancer_vhost: 10.0.0.277# Get a specific member's attributes:78- apache2_mod_proxy:79 balancer_vhost: myws.mydomain.org80 balancer_suffix: /lb/81 member_host: node1.myws.mydomain.org82# Enable all balancer pool members:83- apache2_mod_proxy:84 balancer_vhost: '{{ myloadbalancer_host }}'85 register: result86- apache2_mod_proxy:87 balancer_vhost: '{{ myloadbalancer_host }}'88 member_host: '{{ item.host }}'89 state: present90 with_items: '{{ result.members }}'91# Gracefully disable a member from a loadbalancer node:92- apache2_mod_proxy:93 balancer_vhost: '{{ vhost_host }}'94 member_host: '{{ member.host }}'95 state: drained96 delegate_to: myloadbalancernode97- wait_for:98 host: '{{ member.host }}'99 port: '{{ member.port }}'100 state: drained101 delegate_to: myloadbalancernode102- apache2_mod_proxy:103 balancer_vhost: '{{ vhost_host }}'104 member_host: '{{ member.host }}'105 state: absent106 delegate_to: myloadbalancernode107'''108RETURN = '''109member:110 description: specific balancer member information dictionary, returned when apache2_mod_proxy module is invoked with member_host parameter.111 type: dict112 returned: success113 sample:114 {"attributes":115 {"Busy": "0",116 "Elected": "42",117 "Factor": "1",118 "From": "136K",119 "Load": "0",120 "Route": null,121 "RouteRedir": null,122 "Set": "0",123 "Status": "Init Ok ",124 "To": " 47K",125 "Worker URL": null126 },127 "balancer_url": "http://10.10.0.2/balancer-manager/",128 "host": "10.10.0.20",129 "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.20:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",130 "path": "/ws",131 "port": 8080,132 "protocol": "http",133 "status": {134 "disabled": false,135 "drained": false,136 "hot_standby": false,137 "ignore_errors": false138 }139 }140members:141 description: list of member (defined above) dictionaries, returned when apache2_mod_proxy is invoked with no member_host and state args.142 returned: success143 type: list144 sample:145 [{"attributes": {146 "Busy": "0",147 "Elected": "42",148 "Factor": "1",149 "From": "136K",150 "Load": "0",151 "Route": null,152 "RouteRedir": null,153 "Set": "0",154 "Status": "Init Ok ",155 "To": " 47K",156 "Worker URL": null157 },158 "balancer_url": "http://10.10.0.2/balancer-manager/",159 "host": "10.10.0.20",160 "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.20:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",161 "path": "/ws",162 "port": 8080,163 "protocol": "http",164 "status": {165 "disabled": false,166 "drained": false,167 "hot_standby": false,168 "ignore_errors": false169 }170 },171 {"attributes": {172 "Busy": "0",173 "Elected": "42",174 "Factor": "1",175 "From": "136K",176 "Load": "0",177 "Route": null,178 "RouteRedir": null,179 "Set": "0",180 "Status": "Init Ok ",181 "To": " 47K",182 "Worker URL": null183 },184 "balancer_url": "http://10.10.0.2/balancer-manager/",185 "host": "10.10.0.21",186 "management_url": "http://10.10.0.2/lb/?b=mywsbalancer&w=http://10.10.0.21:8080/ws&nonce=8925436c-79c6-4841-8936-e7d13b79239b",187 "path": "/ws",188 "port": 8080,189 "protocol": "http",190 "status": {191 "disabled": false,192 "drained": false,193 "hot_standby": false,194 "ignore_errors": false}195 }196 ]197'''198import re199try:200 from BeautifulSoup import BeautifulSoup201except ImportError:202 HAS_BEAUTIFULSOUP = False203else:204 HAS_BEAUTIFULSOUP = True205# balancer member attributes extraction regexp:206EXPRESSION = r"(b=([\w\.\-]+)&w=(https?|ajp|wss?|ftp|[sf]cgi)://([\w\.\-]+):?(\d*)([/\w\.\-]*)&?[\w\-\=]*)"207# Apache2 server version extraction regexp:208APACHE_VERSION_EXPRESSION = r"Server Version: Apache/([\d.]+) \(([\w]+)\)"209def regexp_extraction(string, _regexp, groups=1):210 """ Returns the capture group (default=1) specified in the regexp, applied to the string """211 regexp_search = re.search(string=str(string), pattern=str(_regexp))212 if regexp_search:213 if regexp_search.group(groups) != '':214 return str(regexp_search.group(groups))215 return None216class BalancerMember(object):217 """ Apache 2.4 mod_proxy LB balancer member.218 attributes:219 read-only:220 host -> member host (string),221 management_url -> member management url (string),222 protocol -> member protocol (string)223 port -> member port (string),224 path -> member location (string),225 balancer_url -> url of this member's parent balancer (string),226 attributes -> whole member attributes (dictionary)227 module -> ansible module instance (AnsibleModule object).228 writable:229 status -> status of the member (dictionary)230 """231 def __init__(self, management_url, balancer_url, module):232 self.host = regexp_extraction(management_url, str(EXPRESSION), 4)233 self.management_url = str(management_url)234 self.protocol = regexp_extraction(management_url, EXPRESSION, 3)235 self.port = regexp_extraction(management_url, EXPRESSION, 5)236 self.path = regexp_extraction(management_url, EXPRESSION, 6)237 self.balancer_url = str(balancer_url)238 self.module = module239 def get_member_attributes(self):240 """ Returns a dictionary of a balancer member's attributes."""241 balancer_member_page = fetch_url(self.module, self.management_url)242 try:243 assert balancer_member_page[1]['status'] == 200244 except AssertionError:245 self.module.fail_json(msg="Could not get balancer_member_page, check for connectivity! " + balancer_member_page[1])246 else:247 try:248 soup = BeautifulSoup(balancer_member_page[0])249 except TypeError:250 self.module.fail_json(msg="Cannot parse balancer_member_page HTML! " + str(soup))251 else:252 subsoup = soup.findAll('table')[1].findAll('tr')253 keys = subsoup[0].findAll('th')254 for valuesset in subsoup[1::1]:255 if re.search(pattern=self.host, string=str(valuesset)):256 values = valuesset.findAll('td')257 return dict((keys[x].string, values[x].string) for x in range(0, len(keys)))258 def get_member_status(self):259 """ Returns a dictionary of a balancer member's status attributes."""260 status_mapping = {'disabled':'Dis',261 'drained':'Drn',262 'hot_standby':'Stby',263 'ignore_errors':'Ign'}264 status = {}265 actual_status = str(self.attributes['Status'])266 for mode in status_mapping.keys():267 if re.search(pattern=status_mapping[mode], string=actual_status):268 status[mode] = True269 else:270 status[mode] = False271 return status272 def set_member_status(self, values):273 """ Sets a balancer member's status attributes amongst pre-mapped values."""274 values_mapping = {'disabled':'&w_status_D',275 'drained':'&w_status_N',276 'hot_standby':'&w_status_H',277 'ignore_errors':'&w_status_I'}278 request_body = regexp_extraction(self.management_url, EXPRESSION, 1)279 for k in values_mapping.keys():280 if values[str(k)]:281 request_body = request_body + str(values_mapping[k]) + '=1'282 else:283 request_body = request_body + str(values_mapping[k]) + '=0'284 response = fetch_url(self.module, self.management_url, data=str(request_body))285 try:286 assert response[1]['status'] == 200287 except AssertionError:288 self.module.fail_json(msg="Could not set the member status! " + self.host + " " + response[1]['status'])289 attributes = property(get_member_attributes)290 status = property(get_member_status, set_member_status)291class Balancer(object):292 """ Apache httpd 2.4 mod_proxy balancer object"""293 def __init__(self, host, suffix, module, members=None, tls=False):294 if tls:295 self.base_url = str(str('https://') + str(host))296 self.url = str(str('https://') + str(host) + str(suffix))297 else:298 self.base_url = str(str('http://') + str(host))299 self.url = str(str('http://') + str(host) + str(suffix))300 self.module = module301 self.page = self.fetch_balancer_page()302 if members is None:303 self._members = []304 def fetch_balancer_page(self):305 """ Returns the balancer management html page as a string for later parsing."""306 page = fetch_url(self.module, str(self.url))307 try:308 assert page[1]['status'] == 200309 except AssertionError:310 self.module.fail_json(msg="Could not get balancer page! HTTP status response: " + str(page[1]['status']))311 else:312 content = page[0].read()313 apache_version = regexp_extraction(content, APACHE_VERSION_EXPRESSION, 1)314 if not re.search(pattern=r"2\.4\.[\d]*", string=apache_version):315 self.module.fail_json(msg="This module only acts on an Apache2 2.4+ instance, current Apache2 version: " + str(apache_version))316 return content317 def get_balancer_members(self):318 """ Returns members of the balancer as a generator object for later iteration."""319 try:320 soup = BeautifulSoup(self.page)321 except TypeError:322 self.module.fail_json(msg="Cannot parse balancer page HTML! " + str(self.page))323 else:324 for element in soup.findAll('a')[1::1]:325 balancer_member_suffix = str(element.get('href'))326 try:327 assert balancer_member_suffix is not ''328 except AssertionError:329 self.module.fail_json(msg="Argument 'balancer_member_suffix' is empty!")330 else:331 yield BalancerMember(str(self.base_url + balancer_member_suffix), str(self.url), self.module)332 members = property(get_balancer_members)333def main():334 """ Initiates module."""335 module = AnsibleModule(336 argument_spec=dict(337 balancer_vhost=dict(required=True, default=None, type='str'),338 balancer_url_suffix=dict(default="/balancer-manager/", type='str'),339 member_host=dict(type='str'),340 state=dict(type='str'),341 tls=dict(default=False, type='bool'),342 validate_certs=dict(default=True, type='bool')343 ),344 supports_check_mode=True345 )346 if HAS_BEAUTIFULSOUP is False:347 module.fail_json(msg="python module 'BeautifulSoup' is required!")348 if module.params['state'] is not None:349 states = module.params['state'].split(',')350 if (len(states) > 1) and (("present" in states) or ("enabled" in states)):351 module.fail_json(msg="state present/enabled is mutually exclusive with other states!")352 else:353 for _state in states:354 if _state not in ['present', 'absent', 'enabled', 'disabled', 'drained', 'hot_standby', 'ignore_errors']:355 module.fail_json(msg="State can only take values amongst 'present', 'absent', 'enabled', 'disabled', 'drained', 'hot_standby', 'ignore_errors'.")356 else:357 states = ['None']358 mybalancer = Balancer(module.params['balancer_vhost'],359 module.params['balancer_url_suffix'],360 module=module,361 tls=module.params['tls'])362 if module.params['member_host'] is None:363 json_output_list = []364 for member in mybalancer.members:365 json_output_list.append({366 "host": member.host,367 "status": member.status,368 "protocol": member.protocol,369 "port": member.port,370 "path": member.path,371 "attributes": member.attributes,372 "management_url": member.management_url,373 "balancer_url": member.balancer_url374 })375 module.exit_json(376 changed=False,377 members=json_output_list378 )379 else:380 changed = False381 member_exists = False382 member_status = {'disabled': False, 'drained': False, 'hot_standby': False, 'ignore_errors':False}383 for mode in member_status.keys():384 for state in states:385 if mode == state:386 member_status[mode] = True387 elif mode == 'disabled' and state == 'absent':388 member_status[mode] = True389 for member in mybalancer.members:390 if str(member.host) == str(module.params['member_host']):391 member_exists = True392 if module.params['state'] is not None:393 member_status_before = member.status394 if not module.check_mode:395 member_status_after = member.status = member_status396 else:397 member_status_after = member_status398 if member_status_before != member_status_after:399 changed = True400 json_output = {401 "host": member.host,402 "status": member.status,403 "protocol": member.protocol,404 "port": member.port,405 "path": member.path,406 "attributes": member.attributes,407 "management_url": member.management_url,408 "balancer_url": member.balancer_url409 }410 if member_exists:411 module.exit_json(412 changed=changed,413 member=json_output414 )415 else:416 module.fail_json(msg=str(module.params['member_host']) + ' is not a member of the balancer ' + str(module.params['balancer_vhost']) + '!')417from ansible.module_utils.basic import AnsibleModule418from ansible.module_utils.urls import fetch_url419if __name__ == '__main__':...
test_emr_responses.py
Source:test_emr_responses.py
1# Copyright (c) 2010 Jeremy Thurgood <firxen+boto@gmail.com>2#3# Permission is hereby granted, free of charge, to any person obtaining a4# copy of this software and associated documentation files (the5# "Software"), to deal in the Software without restriction, including6# without limitation the rights to use, copy, modify, merge, publish, dis-7# tribute, sublicense, and/or sell copies of the Software, and to permit8# persons to whom the Software is furnished to do so, subject to the fol-9# lowing conditions:10#11# The above copyright notice and this permission notice shall be included12# in all copies or substantial portions of the Software.13#14# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS15# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-16# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT17# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 18# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,19# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS20# IN THE SOFTWARE.21# NOTE: These tests only cover the very simple cases I needed to test22# for the InstanceGroup fix.23import xml.sax24from boto import handler25from boto.emr import emrobject26from boto.resultset import ResultSet27from tests.compat import unittest28JOB_FLOW_EXAMPLE = b"""29<DescribeJobFlowsResponse30 xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-01-15">31 <DescribeJobFlowsResult>32 <JobFlows>33 <member>34 <ExecutionStatusDetail>35 <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>36 <StartDateTime>2009-01-28T21:49:16Z</StartDateTime>37 <State>STARTING</State>38 </ExecutionStatusDetail>39 <BootstrapActions>40 <member>41 <BootstrapActionConfig>42 <ScriptBootstrapAction>43 <Args/>44 <Path>s3://elasticmapreduce/libs/hue/install-hue</Path>45 </ScriptBootstrapAction>46 <Name>Install Hue</Name>47 </BootstrapActionConfig>48 </member>49 </BootstrapActions>50 <VisibleToAllUsers>true</VisibleToAllUsers>51 <SupportedProducts>52 <member>Hue</member>53 </SupportedProducts>54 <Name>MyJobFlowName</Name>55 <LogUri>mybucket/subdir/</LogUri>56 <Steps>57 <member>58 <ExecutionStatusDetail>59 <CreationDateTime>2009-01-28T21:49:16Z</CreationDateTime>60 <State>PENDING</State>61 </ExecutionStatusDetail>62 <StepConfig>63 <HadoopJarStep>64 <Jar>MyJarFile</Jar>65 <MainClass>MyMailClass</MainClass>66 <Args>67 <member>arg1</member>68 <member>arg2</member>69 </Args>70 <Properties/>71 </HadoopJarStep>72 <Name>MyStepName</Name>73 <ActionOnFailure>CONTINUE</ActionOnFailure>74 </StepConfig>75 </member>76 </Steps>77 <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId>78 <Instances>79 <Placement>80 <AvailabilityZone>us-east-1a</AvailabilityZone>81 </Placement>82 <SlaveInstanceType>m1.small</SlaveInstanceType>83 <MasterInstanceType>m1.small</MasterInstanceType>84 <Ec2KeyName>myec2keyname</Ec2KeyName>85 <InstanceCount>4</InstanceCount>86 <KeepJobFlowAliveWhenNoSteps>true</KeepJobFlowAliveWhenNoSteps>87 </Instances>88 </member>89 </JobFlows>90 </DescribeJobFlowsResult>91 <ResponseMetadata>92 <RequestId>9cea3229-ed85-11dd-9877-6fad448a8419</RequestId>93 </ResponseMetadata>94</DescribeJobFlowsResponse>95"""96JOB_FLOW_COMPLETED = b"""97<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">98 <DescribeJobFlowsResult>99 <JobFlows>100 <member>101 <ExecutionStatusDetail>102 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>103 <LastStateChangeReason>Steps completed</LastStateChangeReason>104 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>105 <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>106 <State>COMPLETED</State>107 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>108 </ExecutionStatusDetail>109 <BootstrapActions/>110 <Name>RealJobFlowName</Name>111 <LogUri>s3n://example.emrtest.scripts/jobflow_logs/</LogUri>112 <Steps>113 <member>114 <StepConfig>115 <HadoopJarStep>116 <Jar>s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar</Jar>117 <Args>118 <member>s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch</member>119 </Args>120 <Properties/>121 </HadoopJarStep>122 <Name>Setup Hadoop Debugging</Name>123 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>124 </StepConfig>125 <ExecutionStatusDetail>126 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>127 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>128 <State>COMPLETED</State>129 <EndDateTime>2010-10-21T01:04:22Z</EndDateTime>130 </ExecutionStatusDetail>131 </member>132 <member>133 <StepConfig>134 <HadoopJarStep>135 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>136 <Args>137 <member>-mapper</member>138 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py</member>139 <member>-reducer</member>140 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py</member>141 <member>-input</member>142 <member>s3://example.emrtest.data/raw/2010/10/20/*</member>143 <member>-input</member>144 <member>s3://example.emrtest.data/raw/2010/10/19/*</member>145 <member>-input</member>146 <member>s3://example.emrtest.data/raw/2010/10/18/*</member>147 <member>-input</member>148 <member>s3://example.emrtest.data/raw/2010/10/17/*</member>149 <member>-input</member>150 <member>s3://example.emrtest.data/raw/2010/10/16/*</member>151 <member>-input</member>152 <member>s3://example.emrtest.data/raw/2010/10/15/*</member>153 <member>-input</member>154 <member>s3://example.emrtest.data/raw/2010/10/14/*</member>155 <member>-output</member>156 <member>s3://example.emrtest.crunched/</member>157 </Args>158 <Properties/>159 </HadoopJarStep>160 <Name>testjob_Initial</Name>161 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>162 </StepConfig>163 <ExecutionStatusDetail>164 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>165 <StartDateTime>2010-10-21T01:04:22Z</StartDateTime>166 <State>COMPLETED</State>167 <EndDateTime>2010-10-21T01:36:18Z</EndDateTime>168 </ExecutionStatusDetail>169 </member>170 <member>171 <StepConfig>172 <HadoopJarStep>173 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>174 <Args>175 <member>-mapper</member>176 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py</member>177 <member>-reducer</member>178 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py</member>179 <member>-input</member>180 <member>s3://example.emrtest.crunched/*</member>181 <member>-output</member>182 <member>s3://example.emrtest.step1/</member>183 </Args>184 <Properties/>185 </HadoopJarStep>186 <Name>testjob_step1</Name>187 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>188 </StepConfig>189 <ExecutionStatusDetail>190 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>191 <StartDateTime>2010-10-21T01:36:18Z</StartDateTime>192 <State>COMPLETED</State>193 <EndDateTime>2010-10-21T01:37:51Z</EndDateTime>194 </ExecutionStatusDetail>195 </member>196 <member>197 <StepConfig>198 <HadoopJarStep>199 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>200 <Args>201 <member>-mapper</member>202 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py</member>203 <member>-reducer</member>204 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py</member>205 <member>-input</member>206 <member>s3://example.emrtest.crunched/*</member>207 <member>-output</member>208 <member>s3://example.emrtest.step2/</member>209 </Args>210 <Properties/>211 </HadoopJarStep>212 <Name>testjob_step2</Name>213 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>214 </StepConfig>215 <ExecutionStatusDetail>216 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>217 <StartDateTime>2010-10-21T01:37:51Z</StartDateTime>218 <State>COMPLETED</State>219 <EndDateTime>2010-10-21T01:39:32Z</EndDateTime>220 </ExecutionStatusDetail>221 </member>222 <member>223 <StepConfig>224 <HadoopJarStep>225 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>226 <Args>227 <member>-mapper</member>228 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py</member>229 <member>-reducer</member>230 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py</member>231 <member>-input</member>232 <member>s3://example.emrtest.step1/*</member>233 <member>-output</member>234 <member>s3://example.emrtest.step3/</member>235 </Args>236 <Properties/>237 </HadoopJarStep>238 <Name>testjob_step3</Name>239 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>240 </StepConfig>241 <ExecutionStatusDetail>242 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>243 <StartDateTime>2010-10-21T01:39:32Z</StartDateTime>244 <State>COMPLETED</State>245 <EndDateTime>2010-10-21T01:41:22Z</EndDateTime>246 </ExecutionStatusDetail>247 </member>248 <member>249 <StepConfig>250 <HadoopJarStep>251 <Jar>/home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar</Jar>252 <Args>253 <member>-mapper</member>254 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py</member>255 <member>-reducer</member>256 <member>s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py</member>257 <member>-input</member>258 <member>s3://example.emrtest.step1/*</member>259 <member>-output</member>260 <member>s3://example.emrtest.step4/</member>261 </Args>262 <Properties/>263 </HadoopJarStep>264 <Name>testjob_step4</Name>265 <ActionOnFailure>TERMINATE_JOB_FLOW</ActionOnFailure>266 </StepConfig>267 <ExecutionStatusDetail>268 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>269 <StartDateTime>2010-10-21T01:41:22Z</StartDateTime>270 <State>COMPLETED</State>271 <EndDateTime>2010-10-21T01:43:03Z</EndDateTime>272 </ExecutionStatusDetail>273 </member>274 </Steps>275 <JobFlowId>j-3H3Q13JPFLU22</JobFlowId>276 <Instances>277 <SlaveInstanceType>m1.large</SlaveInstanceType>278 <MasterInstanceId>i-64c21609</MasterInstanceId>279 <Placement>280 <AvailabilityZone>us-east-1b</AvailabilityZone>281 </Placement>282 <InstanceGroups>283 <member>284 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>285 <InstanceRunningCount>0</InstanceRunningCount>286 <StartDateTime>2010-10-21T01:02:09Z</StartDateTime>287 <ReadyDateTime>2010-10-21T01:03:03Z</ReadyDateTime>288 <State>ENDED</State>289 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>290 <InstanceRequestCount>1</InstanceRequestCount>291 <InstanceType>m1.large</InstanceType>292 <Market>ON_DEMAND</Market>293 <LastStateChangeReason>Job flow terminated</LastStateChangeReason>294 <InstanceRole>MASTER</InstanceRole>295 <InstanceGroupId>ig-EVMHOZJ2SCO8</InstanceGroupId>296 <Name>master</Name>297 </member>298 <member>299 <CreationDateTime>2010-10-21T01:00:25Z</CreationDateTime>300 <InstanceRunningCount>0</InstanceRunningCount>301 <StartDateTime>2010-10-21T01:03:59Z</StartDateTime>302 <ReadyDateTime>2010-10-21T01:03:59Z</ReadyDateTime>303 <State>ENDED</State>304 <EndDateTime>2010-10-21T01:44:18Z</EndDateTime>305 <InstanceRequestCount>9</InstanceRequestCount>306 <InstanceType>m1.large</InstanceType>307 <Market>ON_DEMAND</Market>308 <LastStateChangeReason>Job flow terminated</LastStateChangeReason>309 <InstanceRole>CORE</InstanceRole>310 <InstanceGroupId>ig-YZHDYVITVHKB</InstanceGroupId>311 <Name>slave</Name>312 </member>313 </InstanceGroups>314 <NormalizedInstanceHours>40</NormalizedInstanceHours>315 <HadoopVersion>0.20</HadoopVersion>316 <MasterInstanceType>m1.large</MasterInstanceType>317 <MasterPublicDnsName>ec2-184-72-153-139.compute-1.amazonaws.com</MasterPublicDnsName>318 <Ec2KeyName>myubersecurekey</Ec2KeyName>319 <InstanceCount>10</InstanceCount>320 <KeepJobFlowAliveWhenNoSteps>false</KeepJobFlowAliveWhenNoSteps>321 </Instances>322 </member>323 </JobFlows>324 </DescribeJobFlowsResult>325 <ResponseMetadata>326 <RequestId>c31e701d-dcb4-11df-b5d9-337fc7fe4773</RequestId>327 </ResponseMetadata>328</DescribeJobFlowsResponse>329"""330class TestEMRResponses(unittest.TestCase):331 def _parse_xml(self, body, markers):332 rs = ResultSet(markers)333 h = handler.XmlHandler(rs, None)334 xml.sax.parseString(body, h)335 return rs336 def _assert_fields(self, response, **fields):337 for field, expected in fields.items():338 actual = getattr(response, field)339 self.assertEquals(expected, actual,340 "Field %s: %r != %r" % (field, expected, actual))341 def test_JobFlows_example(self):342 [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE,343 [('member', emrobject.JobFlow)])344 self._assert_fields(jobflow,345 creationdatetime='2009-01-28T21:49:16Z',346 startdatetime='2009-01-28T21:49:16Z',347 state='STARTING',348 instancecount='4',349 jobflowid='j-3UN6WX5RRO2AG',350 loguri='mybucket/subdir/',351 name='MyJobFlowName',352 availabilityzone='us-east-1a',353 slaveinstancetype='m1.small',354 masterinstancetype='m1.small',355 ec2keyname='myec2keyname',356 keepjobflowalivewhennosteps='true')357 def test_JobFlows_completed(self):358 [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED,359 [('member', emrobject.JobFlow)])360 self._assert_fields(jobflow,361 creationdatetime='2010-10-21T01:00:25Z',362 startdatetime='2010-10-21T01:03:59Z',363 enddatetime='2010-10-21T01:44:18Z',364 state='COMPLETED',365 instancecount='10',366 jobflowid='j-3H3Q13JPFLU22',367 loguri='s3n://example.emrtest.scripts/jobflow_logs/',368 name='RealJobFlowName',369 availabilityzone='us-east-1b',370 slaveinstancetype='m1.large',371 masterinstancetype='m1.large',372 ec2keyname='myubersecurekey',373 keepjobflowalivewhennosteps='false')374 self.assertEquals(6, len(jobflow.steps))...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run hypothesis automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
