How to use _line_iter method in lisa

Best Python code snippet using lisa_python

other.py

Source:other.py

1# License for Sphinx2# ==================3#4# Copyright (c) 2007-2017 by the Sphinx team (see AUTHORS file).5# All rights reserved.6#7# Redistribution and use in source and binary forms, with or without8# modification, are permitted provided that the following conditions are9# met:10#11# * Redistributions of source code must retain the above copyright12#   notice, this list of conditions and the following disclaimer.13#14# * Redistributions in binary form must reproduce the above copyright15#   notice, this list of conditions and the following disclaimer in the16#   documentation and/or other materials provided with the distribution.17#18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT22# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29from __future__ import absolute_import, print_function30import collections31import re32from typing import (Any, Callable, Deque, Generic, Iterable, Iterator, List,33                    Optional, Tuple, TypeVar, Union, overload)34from six import string_types35from typewriter.docs.parsers import Arg36T = TypeVar('T')37SENTINEL = (None, None)38_directive_regex = re.compile(r'\.\. \S+::')39_google_section_regex = re.compile(r'^(\s|\w)+:\s*$')40_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.+?)\s*\)')41_numpy_section_regex = re.compile(r'^[=\-`:\'"~^_*+#<>]{2,}\s*$')42_xref_regex = re.compile(r'(:\w+:\S+:`.+?`|:\S+:`.+?`|`.+?`)')43_bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')44_enumerated_list_regex = re.compile(45    r'^(?P<paren>\()?'46    r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'47    r'(?(paren)\)|\.)(\s+\S|\s*$)')48class peek_iter(Generic[T]):49    """An iterator object that supports peeking ahead.50    Parameters51    ----------52    obj : Iterable[T]53        `o` is interpreted very differently depending on the presence of54        `sentinel`.55        If `sentinel` is not given, then `o` must be a collection object56        which supports either the iteration protocol or the sequence protocol.57        If `sentinel` is given, then `o` must be a callable object.58    sentinel : Optional[Any]59        If given, the iterator will call `o` with no arguments for each60        call to its `next` method; if the value returned is equal to61        `sentinel`, :exc:`StopIteration` will be raised, otherwise the62        value will be returned.63    See Also64    --------65    `peek_iter` can operate as a drop in replacement for the built-in66    `iter <https://docs.python.org/2/library/functions.html#iter>`_ function.67    Attributes68    ----------69    sentinel70        The value used to indicate the iterator is exhausted. If `sentinel`71        was not given when the `peek_iter` was instantiated, then it will72        be set to a new object instance: ``object()``.73    """74    def __init__(self, obj, sentitnel=None):75        # type: (Iterable[T], Any) -> None76        self._iterable = iter(obj)77        self._cache = collections.deque()  # type: Deque[T]78        if sentitnel is not None:79            self.sentinel = sentitnel80        else:81            self.sentinel = object()82    def __iter__(self):83        # type: () -> Iterator[T]84        return self85    @overload86    def __next__(self):87        # type: () -> T88        pass89    @overload90    def __next__(self, n):91        # type: (int) -> List[T]92        pass93    def __next__(self, n=None):94        # notype95        # note: prevent 2to3 to transform self.next() in next(self) which96        # causes an infinite loop !97        return getattr(self, 'next')(n)98    def _fillcache(self, n):99        """Cache `n` items. If `n` is 0 or None, then 1 item is cached."""100        if not n:101            n = 1102        try:103            while len(self._cache) < n:104                self._cache.append(next(self._iterable))105        except StopIteration:106            while len(self._cache) < n:107                self._cache.append(self.sentinel)108    def has_next(self):109        # type: () -> bool110        """Determine if iterator is exhausted.111        Returns112        -------113        bool114            True if iterator has more items, False otherwise.115        Note116        ----117        Will never raise :exc:`StopIteration`.118        """119        return self.peek() != self.sentinel120    @overload121    def next(self):122        # type: () -> T123        pass124    @overload125    def next(self, n):126        # type: (int) -> List[T]127        pass128    def next(self, n=None):129        # type: (Optional[int]) -> Union[T, List[T]]130        # notype131        """Get the next item or `n` items of the iterator.132        Parameters133        ----------134        n : Optional[int]135            The number of items to retrieve. Defaults to None.136        Returns137        -------138        Union[T, List[T]]139            The next item or `n` items of the iterator. If `n` is None, the140            item itself is returned. If `n` is an int, the items will be141            returned in a list. If `n` is 0, an empty list is returned.142        Raises143        ------144        StopIteration145            Raised if the iterator is exhausted, even if `n` is 0.146        """147        self._fillcache(n)148        if not n:149            if self._cache[0] == self.sentinel:150                raise StopIteration151            if n is None:152                result = self._cache.popleft()  # type: Union[T, List[T]]153            else:154                result = []155        else:156            if self._cache[n - 1] == self.sentinel:157                raise StopIteration158            result = [self._cache.popleft() for i in range(n)]159        return result160    @overload161    def peek(self):162        # type: () -> T163        pass164    @overload165    def peek(self, n):166        # type: (int) -> List[T]167        pass168    def peek(self, n=None):169        # type: (Optional[int]) -> Union[T, List[T]]170        # notype171        """Preview the next item or `n` items of the iterator.172        The iterator is not advanced when peek is called.173        Parameters174        ----------175        n : Optional[int]176        Returns177        -------178        Union[T, List[T]]179            The next item or `n` items of the iterator. If `n` is None, the180            item itself is returned. If `n` is an int, the items will be181            returned in a list. If `n` is 0, an empty list is returned.182            If the iterator is exhausted, `peek_iter.sentinel` is returned,183            or placed as the last item in the returned list.184        Note185        ----186        Will never raise :exc:`StopIteration`.187        """188        self._fillcache(n)189        if n is None:190            result = self._cache[0]  # type: Union[T, List[T]]191        else:192            result = [self._cache[i] for i in range(n)]193        return result194class modify_iter(peek_iter[T]):195    """An iterator object that supports modifying items as they are returned.196    Parameters197    ----------198    obj : Iterable[Any]199        `o` is interpreted very differently depending on the presence of200        `sentinel`.201        If `sentinel` is not given, then `o` must be a collection object202        which supports either the iteration protocol or the sequence protocol.203        If `sentinel` is given, then `o` must be a callable object.204    sentinel : Optional[Any]205        If given, the iterator will call `o` with no arguments for each206        call to its `next` method; if the value returned is equal to207        `sentinel`, :exc:`StopIteration` will be raised, otherwise the208        value will be returned.209    modifier : Callable[[Any], T]210        The function that will be used to modify each item returned by the211        iterator. `modifier` should take a single argument and return a212        single value. Defaults to ``lambda x: x``.213        If `sentinel` is not given, `modifier` must be passed as a keyword214        argument.215    Attributes216    ----------217    modifier : Callable218        `modifier` is called with each item in `o` as it is iterated. The219        return value of `modifier` is returned in lieu of the item.220        Values returned by `peek` as well as `next` are affected by221        `modifier`. However, `modify_iter.sentinel` is never passed through222        `modifier`; it will always be returned from `peek` unmodified.223    Example224    -------225    >>> a = ["     A list    ",226    ...      "   of strings  ",227    ...      "      with     ",228    ...      "      extra    ",229    ...      "   whitespace. "]230    >>> modifier = lambda s: s.strip().replace('with', 'without')231    >>> for s in modify_iter(a, modifier=modifier):232    ...   print('"%s"' % s)233    "A list"234    "of strings"235    "without"236    "extra"237    "whitespace."238    """239    def __init__(self, obj, modifier, sentinel=None):240        # type: (Iterable[Any], Callable[[Any], T], Optional[Any]) -> None241        self.modifier = modifier242        if not callable(self.modifier):243            raise TypeError('modify_iter(o, modifier): '244                            'modifier must be callable')245        super(modify_iter, self).__init__(obj, sentinel)246    def _fillcache(self, n):247        """Cache `n` modified items. If `n` is 0 or None, 1 item is cached.248        Each item returned by the iterator is passed through the249        `modify_iter.modified` function before being cached.250        """251        if not n:252            n = 1253        try:254            while len(self._cache) < n:255                self._cache.append(self.modifier(next(self._iterable)))256        except StopIteration:257            while len(self._cache) < n:258                self._cache.append(self.sentinel)259class GoogleDocstring(object):260    _directive_sections = []  # type: List[str]261    def __init__(self, docstring):262        # type: (Union[str, List[str]]) -> None263        if isinstance(docstring, string_types):264            docstring = docstring.splitlines()265        self._lineno = 1266        self._lines = docstring267        def next_line(line):268            # type: (str) -> Tuple[str, int]269            self._lineno += 1270            return line.rstrip(), self._lineno271        self._line_iter = modify_iter(docstring, next_line, SENTINEL)272        self._is_in_section = False273        self._section_indent = 0274        self._params = None  # type: Optional[List[Tuple[str, Arg]]]275        self._returns = None  # type: Optional[List[Arg]]276        self._yields = None  # type: Optional[List[Arg]]277        self._sections = {278            'args': self._parse_parameters_section,279            'arguments': self._parse_parameters_section,280            'attributes': self._skip_section,281            'example': self._skip_section,282            'examples': self._skip_section,283            'keyword args': self._skip_section,284            'keyword arguments': self._skip_section,285            'methods': self._skip_section,286            'note': self._skip_section,287            'notes': self._skip_section,288            'other parameters': self._skip_section,289            'parameters': self._parse_parameters_section,290            'return': self._parse_returns_section,291            'returns': self._parse_returns_section,292            'raises': self._skip_section,293            'references': self._skip_section,294            'see also': self._skip_section,295            'todo': self._skip_section,296            'warning': self._skip_section,297            'warnings': self._skip_section,298            'warns': self._skip_section,299            'yield': self._parse_yields_section,300            'yields': self._parse_yields_section,301        }302    def _consume_indented_block(self, indent=1):303        # type: (int) -> List[Tuple[str, int]]304        lines = []  # type: List[Tuple[str, int]]305        line = self._line_iter.peek()306        while(not self._is_section_break() and307              (not line[0] or self._is_indented(line[0], indent))):308            lines.append(next(self._line_iter))309            line = self._line_iter.peek()310        return lines311    def _consume_contiguous(self):312        # type: () -> List[Tuple[str, int]]313        lines = []  # type: List[Tuple[str, int]]314        while (self._line_iter.has_next() and315               self._line_iter.peek() and316               not self._is_section_header()):317            lines.append(next(self._line_iter))318        return lines319    def _consume_empty(self):320        # type: () -> List[Tuple[str, int]]321        lines = []  # type: List[Tuple[str, int]]322        line = self._line_iter.peek()323        while self._line_iter.has_next() and not line[0]:324            lines.append(next(self._line_iter))325            line = self._line_iter.peek()326        return lines327    def _consume_field(self, parse_type=True, prefer_type=False):328        # type: (bool, bool) -> Tuple[str, Optional[str], int]329        line, lineno = next(self._line_iter)330        before, colon, after = self._partition_field_on_colon(line, lineno)331        _name = before332        _type = None333        if parse_type:334            match = _google_typed_arg_regex.match(before)335            if match:336                _name = match.group(1)337                _type = match.group(2)338        _name = self._escape_args_and_kwargs(_name)339        if prefer_type and not _type:340            _type, _name = _name, _type341        indent = self._get_indent(line) + 1342        self._consume_indented_block(indent)343        return _name, _type, lineno344    def _consume_fields(self, parse_type=True, prefer_type=False):345        # type: (bool, bool) -> List[Tuple[str, Arg]]346        self._consume_empty()347        fields = []348        while not self._is_section_break():349            _name, _type, lineno = self._consume_field(parse_type, prefer_type)350            if _name or _type:351                fields.append((_name, Arg(_type, lineno)))352        return fields353    def _consume_returns_section(self):354        # type: () -> List[Arg]355        lines = self._dedent(self._consume_to_next_section())356        if lines:357            line, lineno = lines[0]358            before, colon, after = self._partition_field_on_colon(line, lineno)359            _type = None360            if colon:361                match = _google_typed_arg_regex.match(before)362                if match:363                    _type = match.group(2)364                else:365                    _type = before366            if _type:367                return [Arg(_type, lineno)]368        return []369    def _consume_section_header(self):370        # type: () -> str371        section, _ = next(self._line_iter)372        stripped_section = section.strip(':')373        if stripped_section.lower() in self._sections:374            section = stripped_section375        return section376    def _consume_to_end(self):377        # type: () -> List[Tuple[str, int]]378        lines = []  # type: List[Tuple[str, int]]379        while self._line_iter.has_next():380            lines.append(next(self._line_iter))381        return lines382    def _consume_to_next_section(self):383        # type: () -> List[Tuple[str, int]]384        lines = []  # type: List[Tuple[str, int]]385        self._consume_empty()386        while not self._is_section_break():387            lines.append(next(self._line_iter))388        return lines + self._consume_empty()389    def _dedent(self, lines, full=False):390        # type: (List[Tuple[str, int]], bool) -> List[Tuple[str, int]]391        if full:392            return [(line.lstrip(), lineno) for line, lineno in lines]393        else:394            min_indent = self._get_min_indent(lines)395            return [(line[min_indent:], lineno) for line, lineno in lines]396    def _escape_args_and_kwargs(self, name):397        # type: (str) -> str398        if name[:2] == '**':399            return r'\*\*' + name[2:]400        elif name[:1] == '*':401            return r'\*' + name[1:]402        else:403            return name404    def _get_current_indent(self, peek_ahead=0):405        # type: (int) -> int406        line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]407        while line != self._line_iter.sentinel:408            if line[0]:409                return self._get_indent(line[0])410            peek_ahead += 1411            line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]412        return 0413    def _get_indent(self, line):414        # type: (str) -> int415        for i, s in enumerate(line):416            if not s.isspace():417                return i418        return len(line)419    def _get_min_indent(self, lines):420        # type: (List[Tuple[str, int]]) -> int421        min_indent = None422        for line, _ in lines:423            if line:424                indent = self._get_indent(line)425                if min_indent is None:426                    min_indent = indent427                elif indent < min_indent:428                    min_indent = indent429        return min_indent or 0430    def _is_indented(self, line, indent=1):431        # type: (str, int) -> bool432        for i, s in enumerate(line):433            if i >= indent:434                return True435            elif not s.isspace():436                return False437        return False438    def _is_section_header(self):439        # type: () -> bool440        section = self._line_iter.peek()[0].lower()441        match = _google_section_regex.match(section)442        if match and section.strip(':') in self._sections:443            header_indent = self._get_indent(section)444            section_indent = self._get_current_indent(peek_ahead=1)445            return section_indent > header_indent446        elif self._directive_sections:447            if _directive_regex.match(section):448                for directive_section in self._directive_sections:449                    if section.startswith(directive_section):450                        return True451        return False452    def _is_section_break(self):453        # type: () -> bool454        line, _ = self._line_iter.peek()455        return bool(456            not self._line_iter.has_next() or457            self._is_section_header() or458            (self._is_in_section and459                line and460                not self._is_indented(line, self._section_indent)))461    def parse(self):462        # type: () -> Tuple[Optional[List[Tuple[str, Arg]]], Optional[List[Arg]], Optional[List[Arg]]]463        self._consume_empty()464        while self._line_iter.has_next():465            if self._is_section_header():466                try:467                    section = self._consume_section_header()468                    self._is_in_section = True469                    self._section_indent = self._get_current_indent()470                    if _directive_regex.match(section):471                        self._consume_to_next_section()472                    else:473                        self._sections[section.lower()]()474                finally:475                    self._is_in_section = False476                    self._section_indent = 0477            else:478                # FIXME: behavior varied based on parsed lines, which we removed479                # if not self._parsed_lines:480                #     self._consume_contiguous() + self._consume_empty()481                # else:482                self._consume_to_next_section()483        return self._params, self._returns, self._yields484    def _skip_section(self):485        self._consume_to_next_section()486    def _parse_parameters_section(self):487        self._params = self._consume_fields()488    def _parse_returns_section(self):489        self._returns = self._consume_returns_section()490    def _parse_yields_section(self):491        self._yields = self._consume_returns_section()492    def _partition_field_on_colon(self, line, lineno):493        # type: (str, int) -> Tuple[str, str, str]494        before_colon = []495        after_colon = []496        colon = ''497        found_colon = False498        for i, source in enumerate(_xref_regex.split(line)):499            if found_colon:500                after_colon.append(source)501            else:502                if (i % 2) == 0 and ":" in source:503                    found_colon = True504                    before, colon, after = source.partition(":")505                    before_colon.append(before)506                    after_colon.append(after)507                else:508                    before_colon.append(source)509        return ("".join(before_colon).strip(),510                colon,511                "".join(after_colon).strip())512class NumpyDocstring(GoogleDocstring):513    _directive_sections = ['.. index::']514    def _consume_field(self, parse_type=True, prefer_type=False):515        line, lineno = next(self._line_iter)516        if parse_type:517            _name, _, _type = self._partition_field_on_colon(line, lineno)518        else:519            _name = line520            _type = None521        _name, _type = _name.strip(), _type.strip()522        _name = self._escape_args_and_kwargs(_name)523        if prefer_type and not _type:524            _type, _name = _name, _type525        indent = self._get_indent(line) + 1526        self._consume_indented_block(indent)527        return _name, _type, lineno528    def _consume_returns_section(self):529        # type: () -> List[Arg]530        fields = self._consume_fields(prefer_type=True)531        return [x[1] for x in fields]532    def _consume_section_header(self):533        section, _ = next(self._line_iter)534        if not _directive_regex.match(section):535            # Consume the header underline536            next(self._line_iter)537        return section538    def _is_section_break(self):539        line1, line2 = self._line_iter.peek(2)540        return (not self._line_iter.has_next() or541                self._is_section_header() or542                ['', ''] == [line1[0], line2[0]] or543                (self._is_in_section and544                    line1[0] and545                    not self._is_indented(line1[0], self._section_indent)))546    def _is_section_header(self):547        section, underline = self._line_iter.peek(2)548        section = section[0].lower()549        underline = underline[0]550        if section in self._sections and isinstance(underline, string_types):551            return bool(_numpy_section_regex.match(underline))552        elif self._directive_sections:553            if _directive_regex.match(section):554                for directive_section in self._directive_sections:555                    if section.startswith(directive_section):556                        return True557        return False558    _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"...

docstring.py

Source:docstring.py

1# -*- coding: utf-8 -*-2"""3    sphinx.ext.napoleon.docstring4    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~5    Classes for docstring parsing and formatting.6    :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.7    :license: BSD, see LICENSE for details.8"""9import collections10import inspect11import re12# from six import string_types, u13# from six.moves import range14from .iterators import modify_iter15import sys16def _prepare_docstring(s, ignore=1):17    # type: (unicode, int) -> List[unicode]18    """Convert a docstring into lines of parseable reST.  Remove common leading19    indentation, where the indentation of a given number of lines (usually just20    one) is ignored.21    Return the docstring as a list of lines usable for inserting into a docutils22    ViewList (used as argument of nested_parse().)  An empty line is added to23    act as a separator between this docstring and following content.24    """25    lines = s.expandtabs().splitlines()26    # Find minimum indentation of any non-blank lines after ignored lines.27    margin = sys.maxsize28    for line in lines[ignore:]:29        content = len(line.lstrip())30        if content:31            indent = len(line) - content32            margin = min(margin, indent)33    # Remove indentation from ignored lines.34    for i in range(ignore):35        if i < len(lines):36            lines[i] = lines[i].lstrip()37    if margin < sys.maxsize:38        for i in range(ignore, len(lines)):39            lines[i] = lines[i][margin:]40    # Remove any leading blank lines.41    while lines and not lines[0]:42        lines.pop(0)43    # make sure there is an empty line at the end44    if lines and lines[-1]:45        lines.append('')46    return lines47_directive_regex = re.compile(r'\.\. \S+::')48_google_section_regex = re.compile(r'^(\s|\w)+:\s*$')49_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)')50_single_colon_regex = re.compile(r'(?<!:):(?!:)')51_xref_regex = re.compile(r'(:(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)')52_bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')53_enumerated_list_regex = re.compile(54    r'^(?P<paren>\()?'55    r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'56    r'(?(paren)\)|\.)(\s+\S|\s*$)')57class GoogleDocstring(object):58    """Convert Google style docstrings to reStructuredText.59    Parameters60    ----------61    docstring : :obj:`str` or :obj:`list` of :obj:`str`62        The docstring to parse, given either as a string or split into63        individual lines.64    Other Parameters65    ----------------66    what : :obj:`str`, optional67        A string specifying the type of the object to which the docstring68        belongs. Valid values: "module", "class", "exception", "function",69        "method", "attribute".70    name : :obj:`str`, optional71        The fully qualified name of the object.72    obj : module, class, exception, function, method, or attribute73        The object to which the docstring belongs.74    Example75    -------76    >>> from sphinx.ext.napoleon import Config77    >>> config = Config(napoleon_use_param=True, napoleon_use_rtype=True)78    >>> docstring = '''One line summary.79    ...80    ... Extended description.81    ...82    ... Args:83    ...   arg1(int): Description of `arg1`84    ...   arg2(str): Description of `arg2`85    ... Returns:86    ...   str: Description of return value.87    ... '''88    >>> print(GoogleDocstring(docstring, config))89    One line summary.90    <BLANKLINE>91    Extended description.92    <BLANKLINE>93    :param arg1: Description of `arg1`94    :type arg1: int95    :param arg2: Description of `arg2`96    :type arg2: str97    <BLANKLINE>98    :returns: Description of return value.99    :rtype: str100    <BLANKLINE>101    """102    def __init__(self, docstring=None, what='', name='',103                 obj=None, options=None):104        if not what:105            if inspect.isclass(obj):106                what = 'class'107            elif inspect.ismodule(obj):108                what = 'module'109            elif isinstance(obj, collections.Callable):  # type: ignore110                what = 'function'111            else:112                what = 'object'113        if docstring is None:114            if obj is None:115                raise "If docstring is None, obj may not be"116            docstring = obj.__doc__117        self._what = what118        self._name = name119        self._obj = obj120        if isinstance(docstring, str):121            docstring = _prepare_docstring(docstring)122        self._lines = docstring123        self._line_iter = modify_iter(docstring, modifier=lambda s: s.rstrip())124        self._parsed_lines = []  # type: List[unicode]125        self._is_in_section = False126        self._section_indent = 0127        self._directive_sections = []  # type: List[unicode]128        self._entry_sections = {129            'args': self._parse_fields_section,130            'attributes': self._parse_fields_section,131            'returns': self._parse_fields_section,132            'yields': self._parse_fields_section,133            'example args': self._parse_fields_section,134        }  # type: Dict[unicode, Callable]135        self._freeform_sections = {136            'example': self._parse_generic_section,137            'examples': self._parse_generic_section,138            'example returns': self._parse_generic_section,139            'note': self._parse_generic_section,140            'references': self._parse_generic_section,141            'see also': self._parse_generic_section,142            'todo': self._parse_generic_section,143        }  # type: Dict[unicode, Callable]144        self._sections = {145            name: value146            for name, value in [*self._entry_sections.items(), *self._freeform_sections.items()]147        }148        self._parsed_dicts = {149            name: []150            for name in self._entry_sections.keys()151        }152        self._parse()153    def lines(self):154        # type: () -> List[unicode]155        """Return the parsed lines of the docstring in reStructuredText format.156        Returns157        -------158        list(str)159            The lines of the docstring in a list.160        """161        return self._parsed_lines162    def result(self):163        # type: () -> List[unicode]164        """Return the parsed lines of the docstring in reStructuredText format.165        Returns166        -------167        list(str)168            The lines of the docstring in a list.169        """170        return {'sections': self._parsed_lines, **self._parsed_dicts}171    def _consume_indented_block(self, indent=1):172        # type: (int) -> List[unicode]173        lines = []174        line = self._line_iter.peek()175        while(not self._is_section_break() and176              (not line or self._is_indented(line, indent))):177            lines.append(next(self._line_iter))  # type: ignore178            line = self._line_iter.peek()179        return lines180    def _consume_contiguous(self):181        # type: () -> List[unicode]182        lines = []183        while (self._line_iter.has_next() and184               self._line_iter.peek() and185               not self._is_section_header()):186            lines.append(next(self._line_iter))  # type: ignore187        return lines188    def _consume_empty(self):189        # type: () -> List[unicode]190        lines = []191        line = self._line_iter.peek()192        while self._line_iter.has_next() and not line:193            lines.append(next(self._line_iter))  # type: ignore194            line = self._line_iter.peek()195        return lines196    def _consume_field(self, parse_type=True, prefer_type=False):197        # type: (bool, bool) -> Tuple[unicode, unicode, List[unicode]]198        line = next(self._line_iter)  # type: ignore199        before, colon, after = self._partition_field_on_colon(line)200        _name, _type, _desc = before, '', after  # type: unicode, unicode, unicode201        if parse_type:202            match = _google_typed_arg_regex.match(before)  # type: ignore203            if match:204                _name = match.group(1)205                _type = match.group(2)206        _name = self._escape_args_and_kwargs(_name)207        if prefer_type and not _type:208            _type, _name = _name, _type209        indent = self._get_indent(line) + 1210        _descs = [_desc] + self._dedent(self._consume_indented_block(indent))211        return _name, _type, _descs212    def _consume_fields(self, parse_type=True, prefer_type=False):213        # type: (bool, bool) -> List[Tuple[unicode, unicode, List[unicode]]]214        self._consume_empty()215        fields = []216        while not self._is_section_break():217            _name, _type, _desc = self._consume_field(parse_type, prefer_type)218            if _name or _type or _desc:219                fields.append((_name, _type, _desc,))220        return fields221    def _consume_section_header(self):222        # type: () -> unicode223        section = next(self._line_iter)  # type: ignore224        stripped_section = section.strip(':')225        if stripped_section.lower() in self._sections:226            section = stripped_section227        return section228    def _consume_to_end(self):229        # type: () -> List[unicode]230        lines = []231        while self._line_iter.has_next():232            lines.append(next(self._line_iter))  # type: ignore233        return lines234    def _consume_to_next_section(self):235        # type: () -> List[unicode]236        self._consume_empty()237        lines = []238        while not self._is_section_break():239            lines.append(next(self._line_iter))  # type: ignore240        return lines + self._consume_empty()241    def _dedent(self, lines, full=False):242        # type: (List[unicode], bool) -> List[unicode]243        if full:244            return [line.lstrip() for line in lines]245        else:246            min_indent = self._get_min_indent(lines)247            return [line[min_indent:] for line in lines]248    def _escape_args_and_kwargs(self, name):249        # type: (unicode) -> unicode250        if name[:2] == '**':251            return r'\*\*' + name[2:]252        elif name[:1] == '*':253            return r'\*' + name[1:]254        else:255            return name256    def _fix_field_desc(self, desc):257        # type: (List[unicode]) -> List[unicode]258        if self._is_list(desc):259            desc = [u''] + desc260        elif desc[0].endswith('::'):261            desc_block = desc[1:]262            indent = self._get_indent(desc[0])263            block_indent = self._get_initial_indent(desc_block)264            if block_indent > indent:265                desc = [u''] + desc266            else:267                desc = ['', desc[0]] + self._indent(desc_block, 4)268        return desc269    def _get_current_indent(self, peek_ahead=0):270        # type: (int) -> int271        line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]272        while line != self._line_iter.sentinel:273            if line:274                return self._get_indent(line)275            peek_ahead += 1276            line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]277        return 0278    def _get_indent(self, line):279        # type: (unicode) -> int280        for i, s in enumerate(line):281            if not s.isspace():282                return i283        return len(line)284    def _get_initial_indent(self, lines):285        # type: (List[unicode]) -> int286        for line in lines:287            if line:288                return self._get_indent(line)289        return 0290    def _get_min_indent(self, lines):291        # type: (List[unicode]) -> int292        min_indent = None293        for line in lines:294            if line:295                indent = self._get_indent(line)296                if min_indent is None:297                    min_indent = indent298                elif indent < min_indent:299                    min_indent = indent300        return min_indent or 0301    def _indent(self, lines, n=4):302        # type: (List[unicode], int) -> List[unicode]303        return [(' ' * n) + line for line in lines]304    def _is_indented(self, line, indent=1):305        # type: (unicode, int) -> bool306        for i, s in enumerate(line):307            if i >= indent:308                return True309            elif not s.isspace():310                return False311        return False312    def _is_list(self, lines):313        # type: (List[unicode]) -> bool314        if not lines:315            return False316        if _bullet_list_regex.match(lines[0]):  # type: ignore317            return True318        if _enumerated_list_regex.match(lines[0]):  # type: ignore319            return True320        if len(lines) < 2 or lines[0].endswith('::'):321            return False322        indent = self._get_indent(lines[0])323        next_indent = indent324        for line in lines[1:]:325            if line:326                next_indent = self._get_indent(line)327                break328        return next_indent > indent329    def _is_section_header(self):330        # type: () -> bool331        section = self._line_iter.peek().lower()332        match = _google_section_regex.match(section)333        if match and section.strip(':') in self._sections:334            header_indent = self._get_indent(section)335            section_indent = self._get_current_indent(peek_ahead=1)336            return section_indent > header_indent337        elif self._directive_sections:338            if _directive_regex.match(section):339                for directive_section in self._directive_sections:340                    if section.startswith(directive_section):341                        return True342        return False343    def _is_section_break(self):344        # type: () -> bool345        line = self._line_iter.peek()346        return (not self._line_iter.has_next() or347                self._is_section_header() or348                (self._is_in_section and349                    line and350                    not self._is_indented(line, self._section_indent)))351    def _parse(self):352        # type: () -> None353        self._parsed_lines = self._consume_empty()354        while self._line_iter.has_next():355            if self._is_section_header():356                try:357                    section = self._consume_section_header()358                    self._is_in_section = True359                    self._section_indent = self._get_current_indent()360                    if _directive_regex.match(section):  # type: ignore361                        lines = [section] + self._consume_to_next_section()362                    else:363                        section_key = section.lower()364                        parse_section = self._sections[section_key]365                        if section_key in self._parsed_dicts:366                            self._parsed_dicts[section_key].extend(367                                    parse_section())368                        else:369                            self._parsed_lines.append(370                                    (section_key, parse_section()))371                finally:372                    self._is_in_section = False373                    self._section_indent = 0374            else:375                if not self._parsed_lines:376                    self._parsed_lines.append(('text', self._consume_contiguous() + self._consume_empty()))377                else:378                    self._parsed_lines.append(('text', self._consume_to_next_section()))379        # Multiline docstrings often begin right after the """ and then continue380        # with appropriate indentation at the next line break. The above algorithm381        # splits a single text section into two. Merge them here if that happens.382        if len(self._parsed_lines) >= 2:383            first = self._parsed_lines[0]384            second = self._parsed_lines[1]385            if first[0] == 'text' and second[0] == 'text':386                self._parsed_lines = self._parsed_lines[1:]387                self._parsed_lines[0] = ('text', first[1] + second[1])388    def _parse_fields_section(self):389        # type: (unicode) -> List[unicode]390        fields = self._consume_fields()391        # type: (List[Tuple[unicode, unicode, List[unicode]]], unicode, unicode) -> List[unicode]  # NOQA392        lines = []393        for _name, _type, _desc in fields:394            _desc = self._strip_empty(_desc)395            if any(_desc):396                _desc = self._fix_field_desc(_desc)397            lines.append((_name, _type, _desc))398        return lines399    def _parse_generic_section(self):400        # type: (unicode, bool) -> List[unicode]401        lines = self._strip_empty(self._consume_to_next_section())402        lines = self._dedent(lines)403        if lines:404            return lines405        else:406            return ['']407    def _partition_field_on_colon(self, line):408        # type: (unicode) -> Tuple[unicode, unicode, unicode]409        before_colon = []410        after_colon = []411        colon = ''412        found_colon = False413        for i, source in enumerate(_xref_regex.split(line)):  # type: ignore414            if found_colon:415                after_colon.append(source)416            else:417                m = _single_colon_regex.search(source)418                if (i % 2) == 0 and m:419                    found_colon = True420                    colon = source[m.start(): m.end()]421                    before_colon.append(source[:m.start()])422                    after_colon.append(source[m.end():])423                else:424                    before_colon.append(source)425        return ("".join(before_colon).strip(),426                colon,427                "".join(after_colon).strip())428    def _strip_empty(self, lines):429        # type: (List[unicode]) -> List[unicode]430        if lines:431            start = -1432            for i, line in enumerate(lines):433                if line:434                    start = i435                    break436            if start == -1:437                lines = []438            end = -1439            for i in reversed(range(len(lines))):440                line = lines[i]441                if line:442                    end = i443                    break444            if start > 0 or end + 1 < len(lines):445                lines = lines[start:end + 1]...

google_docstring_parser.py

Source:google_docstring_parser.py

1# -*- coding: utf-8 -*-2import logging3import re4from typing import List, Dict, Callable, Tuple5from sphinx.ext.napoleon.iterators import modify_iter6from airflow_munchkin.client_parser.docstring_parser import typehint_parser7from airflow_munchkin.client_parser.docstring_parser.bricks import (8    TypeBrick,9    FieldBrick,10    SectionBrick,11)12_SECTION_REGEX = re.compile(r"^(\s|\w)+:\s*$")13_TYPED_ARG_REGEX = re.compile(r"\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)")14_SINGLE_COLON_REGEX = re.compile(r"(?<!:):(?!:)")15_XREF_REGEX = re.compile(r"(:(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)")16class GoogleDocstringParser:  # pylint: disable=too-few-public-methods17    """Parser Google style docstrings to AST.18    Parameters19    ----------20    docstring : :obj:`str`21        The docstring to parse as a string22    Example23    -------24    >>> docstring = '''One line summary.25    ...26    ... Extended description.27    ...28    ... Args:29    ...   arg1(int): Description of `arg1`30    ...   arg2(str): Description of `arg2`31    ... Returns:32    ...   str: Description of return value.33    ... '''34    >>> sections =  GoogleDocstringParser(docstring).sections()35    >>> import pprint36    >>> pprint.pprint(sections)37    [38    Section(kind='Text', body=['One line summary.', '', 'Extended description.', '']),39    Section(kind='Args', body=[40    Field(name='arg1', kind='int', desc=[Section(kind='Text', body=['Description of `arg1`'])]),41    Field(name='arg2', kind='str', desc=[Section(kind='Text', body=['Description of `arg2`'])])42    ]),43    Section(kind='Returns', body=[44    Field(name='', kind='arg2(str)', desc=[Section(kind='Text', body=['Description of return value.'])])])45    ]46    """47    _name_rgx = re.compile(48        r"^\s*((?::(?P<role>\S+):)?`(?P<name>[a-zA-Z0-9_.-]+)`|"49        r" (?P<name2>[a-zA-Z0-9_.-]+))\s*",50        re.X,51    )52    def __init__(self, docstring: str) -> None:53        lines = docstring.splitlines()54        self._line_iter = modify_iter(lines, modifier=lambda s: s.rstrip())55        self._parsed_sections: List[SectionBrick] = []56        self._is_in_section = False57        self._section_indent = 058        if not hasattr(self, "_sections"):59            self._sections: Dict[str, Callable] = {60                "args": self._parse_parameters_section,61                "arguments": self._parse_parameters_section,62                "parameters": self._parse_parameters_section,63                "return": self._parse_returns_section,64                "returns": self._parse_returns_section,65            }66        self._parse()67    def sections(self) -> List[SectionBrick]:68        """Return the parsed sections of the docstring in reStructuredText format.69        """70        return self._parsed_sections71    def _consume_indented_block(self, indent: int = 1) -> List[str]:72        lines = []73        line = self._line_iter.peek()74        while not self._is_section_break() and (75            not line or self._is_indented(line, indent)76        ):77            lines.append(next(self._line_iter))78            line = self._line_iter.peek()79        return lines80    def _consume_empty(self) -> List[str]:81        lines = []82        line = self._line_iter.peek()83        while self._line_iter.has_next() and not line:84            lines.append(next(self._line_iter))85            line = self._line_iter.peek()86        return lines87    def _consume_field(self) -> Tuple[str, TypeBrick, List[SectionBrick]]:88        line = next(self._line_iter)89        before, _, after = self._partition_field_on_colon(line)90        _name, _type, _desc = before, "", after91        match = _TYPED_ARG_REGEX.match(before)92        if match:93            _name = match.group(1)94            _type = match.group(2)95        _type_element = typehint_parser.parse_typehint(_type)96        indent = self._get_indent(line) + 197        _descs = [_desc] + self._dedent(self._consume_indented_block(indent))98        _sections = [SectionBrick("Text", self._join_sentences(_descs))]99        return _name, _type_element, _sections100    def _consume_fields(self) -> List[FieldBrick]:101        self._consume_empty()102        fields = []103        while not self._is_section_break():104            _name, _type, _desc = self._consume_field()105            if _name or _type or _desc:106                fields.append(FieldBrick(_name, _type, _desc))107        return fields108    def _consume_returns_section(self) -> List[FieldBrick]:109        lines = self._dedent(self._consume_to_next_section())110        if lines:111            before, colon, after = self._partition_field_on_colon(lines[0])112            _name, _type, _desc = "", "", lines113            if colon:114                if after:115                    _desc = [after] + lines[1:]116                else:117                    _desc = lines[1:]118                _type = before119            _sections = [SectionBrick("Text", self._join_sentences(_desc))]120            _type_element = typehint_parser.parse_typehint(_type) if _type else None121            return [FieldBrick(_name, _type_element, _sections)]122        return []123    def _consume_section_header(self) -> str:124        section: str = next(self._line_iter)125        stripped_section = section.strip(":")126        if stripped_section.lower() in self._sections:127            section = stripped_section128        return section129    def _consume_to_next_section(self) -> List[str]:130        self._consume_empty()131        lines = []132        while not self._is_section_break():133            lines.append(next(self._line_iter))134        self._consume_empty()135        return lines136    def _dedent(self, lines: List[str]) -> List[str]:137        min_indent = self._get_min_indent(lines)138        return [line[min_indent:] for line in lines]139    def _get_current_indent(self, peek_ahead: int = 0) -> int:140        line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]141        while line != self._line_iter.sentinel:142            if line:143                return self._get_indent(line)144            peek_ahead += 1145            line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]146        return 0147    @staticmethod148    def _get_indent(line: str) -> int:149        for i, _chr in enumerate(line):150            if not _chr.isspace():151                return i152        return len(line)153    def _get_min_indent(self, lines: List[str]) -> int:154        min_indent = None155        for line in lines:156            if line:157                indent = self._get_indent(line)158                if min_indent is None:159                    min_indent = indent160                elif indent < min_indent:161                    min_indent = indent162        return min_indent or 0163    @staticmethod164    def _is_indented(line: str, indent: int = 1) -> bool:165        for i, _chr in enumerate(line):166            if i >= indent:167                return True168            if not _chr.isspace():169                return False170        return False171    def _is_section_header(self) -> bool:172        section = self._line_iter.peek().lower()173        match = _SECTION_REGEX.match(section)174        if match:175            header_indent = self._get_indent(section)176            section_indent = self._get_current_indent(peek_ahead=1)177            return section_indent > header_indent178        return False179    def _is_section_break(self) -> bool:180        line = self._line_iter.peek()181        return (182            not self._line_iter.has_next()183            or self._is_section_header()184            or (185                self._is_in_section186                and line187                and not self._is_indented(line, self._section_indent)188            )189        )190    def _parse(self) -> None:191        self._parsed_sections = []192        self._consume_empty()193        while self._line_iter.has_next():194            if self._is_section_header():195                try:196                    section = self._consume_section_header()197                    logging.info('Found section: "%s"', section)198                    if not section.lower() in self._sections:199                        logging.info("Skipping section")200                        self._consume_to_next_section()201                        continue202                    logging.info("Parsing section")203                    self._is_in_section = True204                    self._section_indent = self._get_current_indent()205                    sections = self._sections[section.lower()](section)206                finally:207                    self._is_in_section = False208                    self._section_indent = 0209            else:210                sections = SectionBrick(211                    "Text", self._join_sentences(self._consume_to_next_section())212                )213            self._parsed_sections.append(sections)214    def _parse_parameters_section(self, section: str) -> SectionBrick:215        fields = self._consume_fields()216        return SectionBrick(section, fields)217    def _parse_returns_section(self, section: str) -> SectionBrick:218        fields = self._consume_returns_section()219        return SectionBrick(section, fields)220    @staticmethod221    def _join_sentences(lines: List[str]) -> List[str]:222        if not lines:223            return []224        result = []225        sentence = None226        for line in lines:227            if sentence is None:228                sentence = line229            else:230                if line == "":231                    result.append(sentence)232                    sentence = None233                elif line.startswith("- "):234                    sentence += f"\n{line}"235                else:236                    sentence += " " + line.strip()237        if sentence:238            result.append(sentence)239        return result240    @staticmethod241    def _partition_field_on_colon(line: str) -> Tuple[str, str, str]:242        before_colon = []243        after_colon = []244        colon = ""245        found_colon = False246        for i, source in enumerate(_XREF_REGEX.split(line)):247            if found_colon:248                after_colon.append(source)249            else:250                match = _SINGLE_COLON_REGEX.search(source)251                if (i % 2) == 0 and match:252                    found_colon = True253                    colon = source[match.start() : match.end()]  # noqa: E203254                    before_colon.append(source[: match.start()])255                    after_colon.append(source[match.end() :])  # noqa: E203256                else:257                    before_colon.append(source)...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.