Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More
How to use _line_iter method in lisa
Best Python code snippet using lisa_python
other.py
Source:other.py 
1# License for Sphinx2# ==================3#4# Copyright (c) 2007-2017 by the Sphinx team (see AUTHORS file).5# All rights reserved.6#7# Redistribution and use in source and binary forms, with or without8# modification, are permitted provided that the following conditions are9# met:10#11# * Redistributions of source code must retain the above copyright12# notice, this list of conditions and the following disclaimer.13#14# * Redistributions in binary form must reproduce the above copyright15# notice, this list of conditions and the following disclaimer in the16# documentation and/or other materials provided with the distribution.17#18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT22# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.29from __future__ import absolute_import, print_function30import collections31import re32from typing import (Any, Callable, Deque, Generic, Iterable, Iterator, List,33 Optional, Tuple, TypeVar, Union, overload)34from six import string_types35from typewriter.docs.parsers import Arg36T = TypeVar('T')37SENTINEL = (None, None)38_directive_regex = re.compile(r'\.\. \S+::')39_google_section_regex = re.compile(r'^(\s|\w)+:\s*$')40_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.+?)\s*\)')41_numpy_section_regex = re.compile(r'^[=\-`:\'"~^_*+#<>]{2,}\s*$')42_xref_regex = re.compile(r'(:\w+:\S+:`.+?`|:\S+:`.+?`|`.+?`)')43_bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')44_enumerated_list_regex = re.compile(45 r'^(?P<paren>\()?'46 r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'47 r'(?(paren)\)|\.)(\s+\S|\s*$)')48class peek_iter(Generic[T]):49 """An iterator object that supports peeking ahead.50 Parameters51 ----------52 obj : Iterable[T]53 `o` is interpreted very differently depending on the presence of54 `sentinel`.55 If `sentinel` is not given, then `o` must be a collection object56 which supports either the iteration protocol or the sequence protocol.57 If `sentinel` is given, then `o` must be a callable object.58 sentinel : Optional[Any]59 If given, the iterator will call `o` with no arguments for each60 call to its `next` method; if the value returned is equal to61 `sentinel`, :exc:`StopIteration` will be raised, otherwise the62 value will be returned.63 See Also64 --------65 `peek_iter` can operate as a drop in replacement for the built-in66 `iter <https://docs.python.org/2/library/functions.html#iter>`_ function.67 Attributes68 ----------69 sentinel70 The value used to indicate the iterator is exhausted. If `sentinel`71 was not given when the `peek_iter` was instantiated, then it will72 be set to a new object instance: ``object()``.73 """74 def __init__(self, obj, sentitnel=None):75 # type: (Iterable[T], Any) -> None76 self._iterable = iter(obj)77 self._cache = collections.deque() # type: Deque[T]78 if sentitnel is not None:79 self.sentinel = sentitnel80 else:81 self.sentinel = object()82 def __iter__(self):83 # type: () -> Iterator[T]84 return self85 @overload86 def __next__(self):87 # type: () -> T88 pass89 @overload90 def __next__(self, n):91 # type: (int) -> List[T]92 pass93 def __next__(self, n=None):94 # notype95 # note: prevent 2to3 to transform self.next() in next(self) which96 # causes an infinite loop !97 return getattr(self, 'next')(n)98 def _fillcache(self, n):99 """Cache `n` items. If `n` is 0 or None, then 1 item is cached."""100 if not n:101 n = 1102 try:103 while len(self._cache) < n:104 self._cache.append(next(self._iterable))105 except StopIteration:106 while len(self._cache) < n:107 self._cache.append(self.sentinel)108 def has_next(self):109 # type: () -> bool110 """Determine if iterator is exhausted.111 Returns112 -------113 bool114 True if iterator has more items, False otherwise.115 Note116 ----117 Will never raise :exc:`StopIteration`.118 """119 return self.peek() != self.sentinel120 @overload121 def next(self):122 # type: () -> T123 pass124 @overload125 def next(self, n):126 # type: (int) -> List[T]127 pass128 def next(self, n=None):129 # type: (Optional[int]) -> Union[T, List[T]]130 # notype131 """Get the next item or `n` items of the iterator.132 Parameters133 ----------134 n : Optional[int]135 The number of items to retrieve. Defaults to None.136 Returns137 -------138 Union[T, List[T]]139 The next item or `n` items of the iterator. If `n` is None, the140 item itself is returned. If `n` is an int, the items will be141 returned in a list. If `n` is 0, an empty list is returned.142 Raises143 ------144 StopIteration145 Raised if the iterator is exhausted, even if `n` is 0.146 """147 self._fillcache(n)148 if not n:149 if self._cache[0] == self.sentinel:150 raise StopIteration151 if n is None:152 result = self._cache.popleft() # type: Union[T, List[T]]153 else:154 result = []155 else:156 if self._cache[n - 1] == self.sentinel:157 raise StopIteration158 result = [self._cache.popleft() for i in range(n)]159 return result160 @overload161 def peek(self):162 # type: () -> T163 pass164 @overload165 def peek(self, n):166 # type: (int) -> List[T]167 pass168 def peek(self, n=None):169 # type: (Optional[int]) -> Union[T, List[T]]170 # notype171 """Preview the next item or `n` items of the iterator.172 The iterator is not advanced when peek is called.173 Parameters174 ----------175 n : Optional[int]176 Returns177 -------178 Union[T, List[T]]179 The next item or `n` items of the iterator. If `n` is None, the180 item itself is returned. If `n` is an int, the items will be181 returned in a list. If `n` is 0, an empty list is returned.182 If the iterator is exhausted, `peek_iter.sentinel` is returned,183 or placed as the last item in the returned list.184 Note185 ----186 Will never raise :exc:`StopIteration`.187 """188 self._fillcache(n)189 if n is None:190 result = self._cache[0] # type: Union[T, List[T]]191 else:192 result = [self._cache[i] for i in range(n)]193 return result194class modify_iter(peek_iter[T]):195 """An iterator object that supports modifying items as they are returned.196 Parameters197 ----------198 obj : Iterable[Any]199 `o` is interpreted very differently depending on the presence of200 `sentinel`.201 If `sentinel` is not given, then `o` must be a collection object202 which supports either the iteration protocol or the sequence protocol.203 If `sentinel` is given, then `o` must be a callable object.204 sentinel : Optional[Any]205 If given, the iterator will call `o` with no arguments for each206 call to its `next` method; if the value returned is equal to207 `sentinel`, :exc:`StopIteration` will be raised, otherwise the208 value will be returned.209 modifier : Callable[[Any], T]210 The function that will be used to modify each item returned by the211 iterator. `modifier` should take a single argument and return a212 single value. Defaults to ``lambda x: x``.213 If `sentinel` is not given, `modifier` must be passed as a keyword214 argument.215 Attributes216 ----------217 modifier : Callable218 `modifier` is called with each item in `o` as it is iterated. The219 return value of `modifier` is returned in lieu of the item.220 Values returned by `peek` as well as `next` are affected by221 `modifier`. However, `modify_iter.sentinel` is never passed through222 `modifier`; it will always be returned from `peek` unmodified.223 Example224 -------225 >>> a = [" A list ",226 ... " of strings ",227 ... " with ",228 ... " extra ",229 ... " whitespace. "]230 >>> modifier = lambda s: s.strip().replace('with', 'without')231 >>> for s in modify_iter(a, modifier=modifier):232 ... print('"%s"' % s)233 "A list"234 "of strings"235 "without"236 "extra"237 "whitespace."238 """239 def __init__(self, obj, modifier, sentinel=None):240 # type: (Iterable[Any], Callable[[Any], T], Optional[Any]) -> None241 self.modifier = modifier242 if not callable(self.modifier):243 raise TypeError('modify_iter(o, modifier): '244 'modifier must be callable')245 super(modify_iter, self).__init__(obj, sentinel)246 def _fillcache(self, n):247 """Cache `n` modified items. If `n` is 0 or None, 1 item is cached.248 Each item returned by the iterator is passed through the249 `modify_iter.modified` function before being cached.250 """251 if not n:252 n = 1253 try:254 while len(self._cache) < n:255 self._cache.append(self.modifier(next(self._iterable)))256 except StopIteration:257 while len(self._cache) < n:258 self._cache.append(self.sentinel)259class GoogleDocstring(object):260 _directive_sections = [] # type: List[str]261 def __init__(self, docstring):262 # type: (Union[str, List[str]]) -> None263 if isinstance(docstring, string_types):264 docstring = docstring.splitlines()265 self._lineno = 1266 self._lines = docstring267 def next_line(line):268 # type: (str) -> Tuple[str, int]269 self._lineno += 1270 return line.rstrip(), self._lineno271 self._line_iter = modify_iter(docstring, next_line, SENTINEL)272 self._is_in_section = False273 self._section_indent = 0274 self._params = None # type: Optional[List[Tuple[str, Arg]]]275 self._returns = None # type: Optional[List[Arg]]276 self._yields = None # type: Optional[List[Arg]]277 self._sections = {278 'args': self._parse_parameters_section,279 'arguments': self._parse_parameters_section,280 'attributes': self._skip_section,281 'example': self._skip_section,282 'examples': self._skip_section,283 'keyword args': self._skip_section,284 'keyword arguments': self._skip_section,285 'methods': self._skip_section,286 'note': self._skip_section,287 'notes': self._skip_section,288 'other parameters': self._skip_section,289 'parameters': self._parse_parameters_section,290 'return': self._parse_returns_section,291 'returns': self._parse_returns_section,292 'raises': self._skip_section,293 'references': self._skip_section,294 'see also': self._skip_section,295 'todo': self._skip_section,296 'warning': self._skip_section,297 'warnings': self._skip_section,298 'warns': self._skip_section,299 'yield': self._parse_yields_section,300 'yields': self._parse_yields_section,301 }302 def _consume_indented_block(self, indent=1):303 # type: (int) -> List[Tuple[str, int]]304 lines = [] # type: List[Tuple[str, int]]305 line = self._line_iter.peek()306 while(not self._is_section_break() and307 (not line[0] or self._is_indented(line[0], indent))):308 lines.append(next(self._line_iter))309 line = self._line_iter.peek()310 return lines311 def _consume_contiguous(self):312 # type: () -> List[Tuple[str, int]]313 lines = [] # type: List[Tuple[str, int]]314 while (self._line_iter.has_next() and315 self._line_iter.peek() and316 not self._is_section_header()):317 lines.append(next(self._line_iter))318 return lines319 def _consume_empty(self):320 # type: () -> List[Tuple[str, int]]321 lines = [] # type: List[Tuple[str, int]]322 line = self._line_iter.peek()323 while self._line_iter.has_next() and not line[0]:324 lines.append(next(self._line_iter))325 line = self._line_iter.peek()326 return lines327 def _consume_field(self, parse_type=True, prefer_type=False):328 # type: (bool, bool) -> Tuple[str, Optional[str], int]329 line, lineno = next(self._line_iter)330 before, colon, after = self._partition_field_on_colon(line, lineno)331 _name = before332 _type = None333 if parse_type:334 match = _google_typed_arg_regex.match(before)335 if match:336 _name = match.group(1)337 _type = match.group(2)338 _name = self._escape_args_and_kwargs(_name)339 if prefer_type and not _type:340 _type, _name = _name, _type341 indent = self._get_indent(line) + 1342 self._consume_indented_block(indent)343 return _name, _type, lineno344 def _consume_fields(self, parse_type=True, prefer_type=False):345 # type: (bool, bool) -> List[Tuple[str, Arg]]346 self._consume_empty()347 fields = []348 while not self._is_section_break():349 _name, _type, lineno = self._consume_field(parse_type, prefer_type)350 if _name or _type:351 fields.append((_name, Arg(_type, lineno)))352 return fields353 def _consume_returns_section(self):354 # type: () -> List[Arg]355 lines = self._dedent(self._consume_to_next_section())356 if lines:357 line, lineno = lines[0]358 before, colon, after = self._partition_field_on_colon(line, lineno)359 _type = None360 if colon:361 match = _google_typed_arg_regex.match(before)362 if match:363 _type = match.group(2)364 else:365 _type = before366 if _type:367 return [Arg(_type, lineno)]368 return []369 def _consume_section_header(self):370 # type: () -> str371 section, _ = next(self._line_iter)372 stripped_section = section.strip(':')373 if stripped_section.lower() in self._sections:374 section = stripped_section375 return section376 def _consume_to_end(self):377 # type: () -> List[Tuple[str, int]]378 lines = [] # type: List[Tuple[str, int]]379 while self._line_iter.has_next():380 lines.append(next(self._line_iter))381 return lines382 def _consume_to_next_section(self):383 # type: () -> List[Tuple[str, int]]384 lines = [] # type: List[Tuple[str, int]]385 self._consume_empty()386 while not self._is_section_break():387 lines.append(next(self._line_iter))388 return lines + self._consume_empty()389 def _dedent(self, lines, full=False):390 # type: (List[Tuple[str, int]], bool) -> List[Tuple[str, int]]391 if full:392 return [(line.lstrip(), lineno) for line, lineno in lines]393 else:394 min_indent = self._get_min_indent(lines)395 return [(line[min_indent:], lineno) for line, lineno in lines]396 def _escape_args_and_kwargs(self, name):397 # type: (str) -> str398 if name[:2] == '**':399 return r'\*\*' + name[2:]400 elif name[:1] == '*':401 return r'\*' + name[1:]402 else:403 return name404 def _get_current_indent(self, peek_ahead=0):405 # type: (int) -> int406 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]407 while line != self._line_iter.sentinel:408 if line[0]:409 return self._get_indent(line[0])410 peek_ahead += 1411 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]412 return 0413 def _get_indent(self, line):414 # type: (str) -> int415 for i, s in enumerate(line):416 if not s.isspace():417 return i418 return len(line)419 def _get_min_indent(self, lines):420 # type: (List[Tuple[str, int]]) -> int421 min_indent = None422 for line, _ in lines:423 if line:424 indent = self._get_indent(line)425 if min_indent is None:426 min_indent = indent427 elif indent < min_indent:428 min_indent = indent429 return min_indent or 0430 def _is_indented(self, line, indent=1):431 # type: (str, int) -> bool432 for i, s in enumerate(line):433 if i >= indent:434 return True435 elif not s.isspace():436 return False437 return False438 def _is_section_header(self):439 # type: () -> bool440 section = self._line_iter.peek()[0].lower()441 match = _google_section_regex.match(section)442 if match and section.strip(':') in self._sections:443 header_indent = self._get_indent(section)444 section_indent = self._get_current_indent(peek_ahead=1)445 return section_indent > header_indent446 elif self._directive_sections:447 if _directive_regex.match(section):448 for directive_section in self._directive_sections:449 if section.startswith(directive_section):450 return True451 return False452 def _is_section_break(self):453 # type: () -> bool454 line, _ = self._line_iter.peek()455 return bool(456 not self._line_iter.has_next() or457 self._is_section_header() or458 (self._is_in_section and459 line and460 not self._is_indented(line, self._section_indent)))461 def parse(self):462 # type: () -> Tuple[Optional[List[Tuple[str, Arg]]], Optional[List[Arg]], Optional[List[Arg]]]463 self._consume_empty()464 while self._line_iter.has_next():465 if self._is_section_header():466 try:467 section = self._consume_section_header()468 self._is_in_section = True469 self._section_indent = self._get_current_indent()470 if _directive_regex.match(section):471 self._consume_to_next_section()472 else:473 self._sections[section.lower()]()474 finally:475 self._is_in_section = False476 self._section_indent = 0477 else:478 # FIXME: behavior varied based on parsed lines, which we removed479 # if not self._parsed_lines:480 # self._consume_contiguous() + self._consume_empty()481 # else:482 self._consume_to_next_section()483 return self._params, self._returns, self._yields484 def _skip_section(self):485 self._consume_to_next_section()486 def _parse_parameters_section(self):487 self._params = self._consume_fields()488 def _parse_returns_section(self):489 self._returns = self._consume_returns_section()490 def _parse_yields_section(self):491 self._yields = self._consume_returns_section()492 def _partition_field_on_colon(self, line, lineno):493 # type: (str, int) -> Tuple[str, str, str]494 before_colon = []495 after_colon = []496 colon = ''497 found_colon = False498 for i, source in enumerate(_xref_regex.split(line)):499 if found_colon:500 after_colon.append(source)501 else:502 if (i % 2) == 0 and ":" in source:503 found_colon = True504 before, colon, after = source.partition(":")505 before_colon.append(before)506 after_colon.append(after)507 else:508 before_colon.append(source)509 return ("".join(before_colon).strip(),510 colon,511 "".join(after_colon).strip())512class NumpyDocstring(GoogleDocstring):513 _directive_sections = ['.. index::']514 def _consume_field(self, parse_type=True, prefer_type=False):515 line, lineno = next(self._line_iter)516 if parse_type:517 _name, _, _type = self._partition_field_on_colon(line, lineno)518 else:519 _name = line520 _type = None521 _name, _type = _name.strip(), _type.strip()522 _name = self._escape_args_and_kwargs(_name)523 if prefer_type and not _type:524 _type, _name = _name, _type525 indent = self._get_indent(line) + 1526 self._consume_indented_block(indent)527 return _name, _type, lineno528 def _consume_returns_section(self):529 # type: () -> List[Arg]530 fields = self._consume_fields(prefer_type=True)531 return [x[1] for x in fields]532 def _consume_section_header(self):533 section, _ = next(self._line_iter)534 if not _directive_regex.match(section):535 # Consume the header underline536 next(self._line_iter)537 return section538 def _is_section_break(self):539 line1, line2 = self._line_iter.peek(2)540 return (not self._line_iter.has_next() or541 self._is_section_header() or542 ['', ''] == [line1[0], line2[0]] or543 (self._is_in_section and544 line1[0] and545 not self._is_indented(line1[0], self._section_indent)))546 def _is_section_header(self):547 section, underline = self._line_iter.peek(2)548 section = section[0].lower()549 underline = underline[0]550 if section in self._sections and isinstance(underline, string_types):551 return bool(_numpy_section_regex.match(underline))552 elif self._directive_sections:553 if _directive_regex.match(section):554 for directive_section in self._directive_sections:555 if section.startswith(directive_section):556 return True557 return False558 _name_rgx = re.compile(r"^\s*(:(?P<role>\w+):`(?P<name>[a-zA-Z0-9_.-]+)`|"...
docstring.py
Source:docstring.py
1# -*- coding: utf-8 -*-2"""3 sphinx.ext.napoleon.docstring4 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~5 Classes for docstring parsing and formatting.6 :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS.7 :license: BSD, see LICENSE for details.8"""9import collections10import inspect11import re12# from six import string_types, u13# from six.moves import range14from .iterators import modify_iter15import sys16def _prepare_docstring(s, ignore=1):17 # type: (unicode, int) -> List[unicode]18 """Convert a docstring into lines of parseable reST. Remove common leading19 indentation, where the indentation of a given number of lines (usually just20 one) is ignored.21 Return the docstring as a list of lines usable for inserting into a docutils22 ViewList (used as argument of nested_parse().) An empty line is added to23 act as a separator between this docstring and following content.24 """25 lines = s.expandtabs().splitlines()26 # Find minimum indentation of any non-blank lines after ignored lines.27 margin = sys.maxsize28 for line in lines[ignore:]:29 content = len(line.lstrip())30 if content:31 indent = len(line) - content32 margin = min(margin, indent)33 # Remove indentation from ignored lines.34 for i in range(ignore):35 if i < len(lines):36 lines[i] = lines[i].lstrip()37 if margin < sys.maxsize:38 for i in range(ignore, len(lines)):39 lines[i] = lines[i][margin:]40 # Remove any leading blank lines.41 while lines and not lines[0]:42 lines.pop(0)43 # make sure there is an empty line at the end44 if lines and lines[-1]:45 lines.append('')46 return lines47_directive_regex = re.compile(r'\.\. \S+::')48_google_section_regex = re.compile(r'^(\s|\w)+:\s*$')49_google_typed_arg_regex = re.compile(r'\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)')50_single_colon_regex = re.compile(r'(?<!:):(?!:)')51_xref_regex = re.compile(r'(:(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)')52_bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')53_enumerated_list_regex = re.compile(54 r'^(?P<paren>\()?'55 r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'56 r'(?(paren)\)|\.)(\s+\S|\s*$)')57class GoogleDocstring(object):58 """Convert Google style docstrings to reStructuredText.59 Parameters60 ----------61 docstring : :obj:`str` or :obj:`list` of :obj:`str`62 The docstring to parse, given either as a string or split into63 individual lines.64 Other Parameters65 ----------------66 what : :obj:`str`, optional67 A string specifying the type of the object to which the docstring68 belongs. Valid values: "module", "class", "exception", "function",69 "method", "attribute".70 name : :obj:`str`, optional71 The fully qualified name of the object.72 obj : module, class, exception, function, method, or attribute73 The object to which the docstring belongs.74 Example75 -------76 >>> from sphinx.ext.napoleon import Config77 >>> config = Config(napoleon_use_param=True, napoleon_use_rtype=True)78 >>> docstring = '''One line summary.79 ...80 ... Extended description.81 ...82 ... Args:83 ... arg1(int): Description of `arg1`84 ... arg2(str): Description of `arg2`85 ... Returns:86 ... str: Description of return value.87 ... '''88 >>> print(GoogleDocstring(docstring, config))89 One line summary.90 <BLANKLINE>91 Extended description.92 <BLANKLINE>93 :param arg1: Description of `arg1`94 :type arg1: int95 :param arg2: Description of `arg2`96 :type arg2: str97 <BLANKLINE>98 :returns: Description of return value.99 :rtype: str100 <BLANKLINE>101 """102 def __init__(self, docstring=None, what='', name='',103 obj=None, options=None):104 if not what:105 if inspect.isclass(obj):106 what = 'class'107 elif inspect.ismodule(obj):108 what = 'module'109 elif isinstance(obj, collections.Callable): # type: ignore110 what = 'function'111 else:112 what = 'object'113 if docstring is None:114 if obj is None:115 raise "If docstring is None, obj may not be"116 docstring = obj.__doc__117 self._what = what118 self._name = name119 self._obj = obj120 if isinstance(docstring, str):121 docstring = _prepare_docstring(docstring)122 self._lines = docstring123 self._line_iter = modify_iter(docstring, modifier=lambda s: s.rstrip())124 self._parsed_lines = [] # type: List[unicode]125 self._is_in_section = False126 self._section_indent = 0127 self._directive_sections = [] # type: List[unicode]128 self._entry_sections = {129 'args': self._parse_fields_section,130 'attributes': self._parse_fields_section,131 'returns': self._parse_fields_section,132 'yields': self._parse_fields_section,133 'example args': self._parse_fields_section,134 } # type: Dict[unicode, Callable]135 self._freeform_sections = {136 'example': self._parse_generic_section,137 'examples': self._parse_generic_section,138 'example returns': self._parse_generic_section,139 'note': self._parse_generic_section,140 'references': self._parse_generic_section,141 'see also': self._parse_generic_section,142 'todo': self._parse_generic_section,143 } # type: Dict[unicode, Callable]144 self._sections = {145 name: value146 for name, value in [*self._entry_sections.items(), *self._freeform_sections.items()]147 }148 self._parsed_dicts = {149 name: []150 for name in self._entry_sections.keys()151 }152 self._parse()153 def lines(self):154 # type: () -> List[unicode]155 """Return the parsed lines of the docstring in reStructuredText format.156 Returns157 -------158 list(str)159 The lines of the docstring in a list.160 """161 return self._parsed_lines162 def result(self):163 # type: () -> List[unicode]164 """Return the parsed lines of the docstring in reStructuredText format.165 Returns166 -------167 list(str)168 The lines of the docstring in a list.169 """170 return {'sections': self._parsed_lines, **self._parsed_dicts}171 def _consume_indented_block(self, indent=1):172 # type: (int) -> List[unicode]173 lines = []174 line = self._line_iter.peek()175 while(not self._is_section_break() and176 (not line or self._is_indented(line, indent))):177 lines.append(next(self._line_iter)) # type: ignore178 line = self._line_iter.peek()179 return lines180 def _consume_contiguous(self):181 # type: () -> List[unicode]182 lines = []183 while (self._line_iter.has_next() and184 self._line_iter.peek() and185 not self._is_section_header()):186 lines.append(next(self._line_iter)) # type: ignore187 return lines188 def _consume_empty(self):189 # type: () -> List[unicode]190 lines = []191 line = self._line_iter.peek()192 while self._line_iter.has_next() and not line:193 lines.append(next(self._line_iter)) # type: ignore194 line = self._line_iter.peek()195 return lines196 def _consume_field(self, parse_type=True, prefer_type=False):197 # type: (bool, bool) -> Tuple[unicode, unicode, List[unicode]]198 line = next(self._line_iter) # type: ignore199 before, colon, after = self._partition_field_on_colon(line)200 _name, _type, _desc = before, '', after # type: unicode, unicode, unicode201 if parse_type:202 match = _google_typed_arg_regex.match(before) # type: ignore203 if match:204 _name = match.group(1)205 _type = match.group(2)206 _name = self._escape_args_and_kwargs(_name)207 if prefer_type and not _type:208 _type, _name = _name, _type209 indent = self._get_indent(line) + 1210 _descs = [_desc] + self._dedent(self._consume_indented_block(indent))211 return _name, _type, _descs212 def _consume_fields(self, parse_type=True, prefer_type=False):213 # type: (bool, bool) -> List[Tuple[unicode, unicode, List[unicode]]]214 self._consume_empty()215 fields = []216 while not self._is_section_break():217 _name, _type, _desc = self._consume_field(parse_type, prefer_type)218 if _name or _type or _desc:219 fields.append((_name, _type, _desc,))220 return fields221 def _consume_section_header(self):222 # type: () -> unicode223 section = next(self._line_iter) # type: ignore224 stripped_section = section.strip(':')225 if stripped_section.lower() in self._sections:226 section = stripped_section227 return section228 def _consume_to_end(self):229 # type: () -> List[unicode]230 lines = []231 while self._line_iter.has_next():232 lines.append(next(self._line_iter)) # type: ignore233 return lines234 def _consume_to_next_section(self):235 # type: () -> List[unicode]236 self._consume_empty()237 lines = []238 while not self._is_section_break():239 lines.append(next(self._line_iter)) # type: ignore240 return lines + self._consume_empty()241 def _dedent(self, lines, full=False):242 # type: (List[unicode], bool) -> List[unicode]243 if full:244 return [line.lstrip() for line in lines]245 else:246 min_indent = self._get_min_indent(lines)247 return [line[min_indent:] for line in lines]248 def _escape_args_and_kwargs(self, name):249 # type: (unicode) -> unicode250 if name[:2] == '**':251 return r'\*\*' + name[2:]252 elif name[:1] == '*':253 return r'\*' + name[1:]254 else:255 return name256 def _fix_field_desc(self, desc):257 # type: (List[unicode]) -> List[unicode]258 if self._is_list(desc):259 desc = [u''] + desc260 elif desc[0].endswith('::'):261 desc_block = desc[1:]262 indent = self._get_indent(desc[0])263 block_indent = self._get_initial_indent(desc_block)264 if block_indent > indent:265 desc = [u''] + desc266 else:267 desc = ['', desc[0]] + self._indent(desc_block, 4)268 return desc269 def _get_current_indent(self, peek_ahead=0):270 # type: (int) -> int271 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]272 while line != self._line_iter.sentinel:273 if line:274 return self._get_indent(line)275 peek_ahead += 1276 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]277 return 0278 def _get_indent(self, line):279 # type: (unicode) -> int280 for i, s in enumerate(line):281 if not s.isspace():282 return i283 return len(line)284 def _get_initial_indent(self, lines):285 # type: (List[unicode]) -> int286 for line in lines:287 if line:288 return self._get_indent(line)289 return 0290 def _get_min_indent(self, lines):291 # type: (List[unicode]) -> int292 min_indent = None293 for line in lines:294 if line:295 indent = self._get_indent(line)296 if min_indent is None:297 min_indent = indent298 elif indent < min_indent:299 min_indent = indent300 return min_indent or 0301 def _indent(self, lines, n=4):302 # type: (List[unicode], int) -> List[unicode]303 return [(' ' * n) + line for line in lines]304 def _is_indented(self, line, indent=1):305 # type: (unicode, int) -> bool306 for i, s in enumerate(line):307 if i >= indent:308 return True309 elif not s.isspace():310 return False311 return False312 def _is_list(self, lines):313 # type: (List[unicode]) -> bool314 if not lines:315 return False316 if _bullet_list_regex.match(lines[0]): # type: ignore317 return True318 if _enumerated_list_regex.match(lines[0]): # type: ignore319 return True320 if len(lines) < 2 or lines[0].endswith('::'):321 return False322 indent = self._get_indent(lines[0])323 next_indent = indent324 for line in lines[1:]:325 if line:326 next_indent = self._get_indent(line)327 break328 return next_indent > indent329 def _is_section_header(self):330 # type: () -> bool331 section = self._line_iter.peek().lower()332 match = _google_section_regex.match(section)333 if match and section.strip(':') in self._sections:334 header_indent = self._get_indent(section)335 section_indent = self._get_current_indent(peek_ahead=1)336 return section_indent > header_indent337 elif self._directive_sections:338 if _directive_regex.match(section):339 for directive_section in self._directive_sections:340 if section.startswith(directive_section):341 return True342 return False343 def _is_section_break(self):344 # type: () -> bool345 line = self._line_iter.peek()346 return (not self._line_iter.has_next() or347 self._is_section_header() or348 (self._is_in_section and349 line and350 not self._is_indented(line, self._section_indent)))351 def _parse(self):352 # type: () -> None353 self._parsed_lines = self._consume_empty()354 while self._line_iter.has_next():355 if self._is_section_header():356 try:357 section = self._consume_section_header()358 self._is_in_section = True359 self._section_indent = self._get_current_indent()360 if _directive_regex.match(section): # type: ignore361 lines = [section] + self._consume_to_next_section()362 else:363 section_key = section.lower()364 parse_section = self._sections[section_key]365 if section_key in self._parsed_dicts:366 self._parsed_dicts[section_key].extend(367 parse_section())368 else:369 self._parsed_lines.append(370 (section_key, parse_section()))371 finally:372 self._is_in_section = False373 self._section_indent = 0374 else:375 if not self._parsed_lines:376 self._parsed_lines.append(('text', self._consume_contiguous() + self._consume_empty()))377 else:378 self._parsed_lines.append(('text', self._consume_to_next_section()))379 # Multiline docstrings often begin right after the """ and then continue380 # with appropriate indentation at the next line break. The above algorithm381 # splits a single text section into two. Merge them here if that happens.382 if len(self._parsed_lines) >= 2:383 first = self._parsed_lines[0]384 second = self._parsed_lines[1]385 if first[0] == 'text' and second[0] == 'text':386 self._parsed_lines = self._parsed_lines[1:]387 self._parsed_lines[0] = ('text', first[1] + second[1])388 def _parse_fields_section(self):389 # type: (unicode) -> List[unicode]390 fields = self._consume_fields()391 # type: (List[Tuple[unicode, unicode, List[unicode]]], unicode, unicode) -> List[unicode] # NOQA392 lines = []393 for _name, _type, _desc in fields:394 _desc = self._strip_empty(_desc)395 if any(_desc):396 _desc = self._fix_field_desc(_desc)397 lines.append((_name, _type, _desc))398 return lines399 def _parse_generic_section(self):400 # type: (unicode, bool) -> List[unicode]401 lines = self._strip_empty(self._consume_to_next_section())402 lines = self._dedent(lines)403 if lines:404 return lines405 else:406 return ['']407 def _partition_field_on_colon(self, line):408 # type: (unicode) -> Tuple[unicode, unicode, unicode]409 before_colon = []410 after_colon = []411 colon = ''412 found_colon = False413 for i, source in enumerate(_xref_regex.split(line)): # type: ignore414 if found_colon:415 after_colon.append(source)416 else:417 m = _single_colon_regex.search(source)418 if (i % 2) == 0 and m:419 found_colon = True420 colon = source[m.start(): m.end()]421 before_colon.append(source[:m.start()])422 after_colon.append(source[m.end():])423 else:424 before_colon.append(source)425 return ("".join(before_colon).strip(),426 colon,427 "".join(after_colon).strip())428 def _strip_empty(self, lines):429 # type: (List[unicode]) -> List[unicode]430 if lines:431 start = -1432 for i, line in enumerate(lines):433 if line:434 start = i435 break436 if start == -1:437 lines = []438 end = -1439 for i in reversed(range(len(lines))):440 line = lines[i]441 if line:442 end = i443 break444 if start > 0 or end + 1 < len(lines):445 lines = lines[start:end + 1]...
google_docstring_parser.py
Source:google_docstring_parser.py
1# -*- coding: utf-8 -*-2import logging3import re4from typing import List, Dict, Callable, Tuple5from sphinx.ext.napoleon.iterators import modify_iter6from airflow_munchkin.client_parser.docstring_parser import typehint_parser7from airflow_munchkin.client_parser.docstring_parser.bricks import (8 TypeBrick,9 FieldBrick,10 SectionBrick,11)12_SECTION_REGEX = re.compile(r"^(\s|\w)+:\s*$")13_TYPED_ARG_REGEX = re.compile(r"\s*(.+?)\s*\(\s*(.*[^\s]+)\s*\)")14_SINGLE_COLON_REGEX = re.compile(r"(?<!:):(?!:)")15_XREF_REGEX = re.compile(r"(:(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)")16class GoogleDocstringParser: # pylint: disable=too-few-public-methods17 """Parser Google style docstrings to AST.18 Parameters19 ----------20 docstring : :obj:`str`21 The docstring to parse as a string22 Example23 -------24 >>> docstring = '''One line summary.25 ...26 ... Extended description.27 ...28 ... Args:29 ... arg1(int): Description of `arg1`30 ... arg2(str): Description of `arg2`31 ... Returns:32 ... str: Description of return value.33 ... '''34 >>> sections = GoogleDocstringParser(docstring).sections()35 >>> import pprint36 >>> pprint.pprint(sections)37 [38 Section(kind='Text', body=['One line summary.', '', 'Extended description.', '']),39 Section(kind='Args', body=[40 Field(name='arg1', kind='int', desc=[Section(kind='Text', body=['Description of `arg1`'])]),41 Field(name='arg2', kind='str', desc=[Section(kind='Text', body=['Description of `arg2`'])])42 ]),43 Section(kind='Returns', body=[44 Field(name='', kind='arg2(str)', desc=[Section(kind='Text', body=['Description of return value.'])])])45 ]46 """47 _name_rgx = re.compile(48 r"^\s*((?::(?P<role>\S+):)?`(?P<name>[a-zA-Z0-9_.-]+)`|"49 r" (?P<name2>[a-zA-Z0-9_.-]+))\s*",50 re.X,51 )52 def __init__(self, docstring: str) -> None:53 lines = docstring.splitlines()54 self._line_iter = modify_iter(lines, modifier=lambda s: s.rstrip())55 self._parsed_sections: List[SectionBrick] = []56 self._is_in_section = False57 self._section_indent = 058 if not hasattr(self, "_sections"):59 self._sections: Dict[str, Callable] = {60 "args": self._parse_parameters_section,61 "arguments": self._parse_parameters_section,62 "parameters": self._parse_parameters_section,63 "return": self._parse_returns_section,64 "returns": self._parse_returns_section,65 }66 self._parse()67 def sections(self) -> List[SectionBrick]:68 """Return the parsed sections of the docstring in reStructuredText format.69 """70 return self._parsed_sections71 def _consume_indented_block(self, indent: int = 1) -> List[str]:72 lines = []73 line = self._line_iter.peek()74 while not self._is_section_break() and (75 not line or self._is_indented(line, indent)76 ):77 lines.append(next(self._line_iter))78 line = self._line_iter.peek()79 return lines80 def _consume_empty(self) -> List[str]:81 lines = []82 line = self._line_iter.peek()83 while self._line_iter.has_next() and not line:84 lines.append(next(self._line_iter))85 line = self._line_iter.peek()86 return lines87 def _consume_field(self) -> Tuple[str, TypeBrick, List[SectionBrick]]:88 line = next(self._line_iter)89 before, _, after = self._partition_field_on_colon(line)90 _name, _type, _desc = before, "", after91 match = _TYPED_ARG_REGEX.match(before)92 if match:93 _name = match.group(1)94 _type = match.group(2)95 _type_element = typehint_parser.parse_typehint(_type)96 indent = self._get_indent(line) + 197 _descs = [_desc] + self._dedent(self._consume_indented_block(indent))98 _sections = [SectionBrick("Text", self._join_sentences(_descs))]99 return _name, _type_element, _sections100 def _consume_fields(self) -> List[FieldBrick]:101 self._consume_empty()102 fields = []103 while not self._is_section_break():104 _name, _type, _desc = self._consume_field()105 if _name or _type or _desc:106 fields.append(FieldBrick(_name, _type, _desc))107 return fields108 def _consume_returns_section(self) -> List[FieldBrick]:109 lines = self._dedent(self._consume_to_next_section())110 if lines:111 before, colon, after = self._partition_field_on_colon(lines[0])112 _name, _type, _desc = "", "", lines113 if colon:114 if after:115 _desc = [after] + lines[1:]116 else:117 _desc = lines[1:]118 _type = before119 _sections = [SectionBrick("Text", self._join_sentences(_desc))]120 _type_element = typehint_parser.parse_typehint(_type) if _type else None121 return [FieldBrick(_name, _type_element, _sections)]122 return []123 def _consume_section_header(self) -> str:124 section: str = next(self._line_iter)125 stripped_section = section.strip(":")126 if stripped_section.lower() in self._sections:127 section = stripped_section128 return section129 def _consume_to_next_section(self) -> List[str]:130 self._consume_empty()131 lines = []132 while not self._is_section_break():133 lines.append(next(self._line_iter))134 self._consume_empty()135 return lines136 def _dedent(self, lines: List[str]) -> List[str]:137 min_indent = self._get_min_indent(lines)138 return [line[min_indent:] for line in lines]139 def _get_current_indent(self, peek_ahead: int = 0) -> int:140 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]141 while line != self._line_iter.sentinel:142 if line:143 return self._get_indent(line)144 peek_ahead += 1145 line = self._line_iter.peek(peek_ahead + 1)[peek_ahead]146 return 0147 @staticmethod148 def _get_indent(line: str) -> int:149 for i, _chr in enumerate(line):150 if not _chr.isspace():151 return i152 return len(line)153 def _get_min_indent(self, lines: List[str]) -> int:154 min_indent = None155 for line in lines:156 if line:157 indent = self._get_indent(line)158 if min_indent is None:159 min_indent = indent160 elif indent < min_indent:161 min_indent = indent162 return min_indent or 0163 @staticmethod164 def _is_indented(line: str, indent: int = 1) -> bool:165 for i, _chr in enumerate(line):166 if i >= indent:167 return True168 if not _chr.isspace():169 return False170 return False171 def _is_section_header(self) -> bool:172 section = self._line_iter.peek().lower()173 match = _SECTION_REGEX.match(section)174 if match:175 header_indent = self._get_indent(section)176 section_indent = self._get_current_indent(peek_ahead=1)177 return section_indent > header_indent178 return False179 def _is_section_break(self) -> bool:180 line = self._line_iter.peek()181 return (182 not self._line_iter.has_next()183 or self._is_section_header()184 or (185 self._is_in_section186 and line187 and not self._is_indented(line, self._section_indent)188 )189 )190 def _parse(self) -> None:191 self._parsed_sections = []192 self._consume_empty()193 while self._line_iter.has_next():194 if self._is_section_header():195 try:196 section = self._consume_section_header()197 logging.info('Found section: "%s"', section)198 if not section.lower() in self._sections:199 logging.info("Skipping section")200 self._consume_to_next_section()201 continue202 logging.info("Parsing section")203 self._is_in_section = True204 self._section_indent = self._get_current_indent()205 sections = self._sections[section.lower()](section)206 finally:207 self._is_in_section = False208 self._section_indent = 0209 else:210 sections = SectionBrick(211 "Text", self._join_sentences(self._consume_to_next_section())212 )213 self._parsed_sections.append(sections)214 def _parse_parameters_section(self, section: str) -> SectionBrick:215 fields = self._consume_fields()216 return SectionBrick(section, fields)217 def _parse_returns_section(self, section: str) -> SectionBrick:218 fields = self._consume_returns_section()219 return SectionBrick(section, fields)220 @staticmethod221 def _join_sentences(lines: List[str]) -> List[str]:222 if not lines:223 return []224 result = []225 sentence = None226 for line in lines:227 if sentence is None:228 sentence = line229 else:230 if line == "":231 result.append(sentence)232 sentence = None233 elif line.startswith("- "):234 sentence += f"\n{line}"235 else:236 sentence += " " + line.strip()237 if sentence:238 result.append(sentence)239 return result240 @staticmethod241 def _partition_field_on_colon(line: str) -> Tuple[str, str, str]:242 before_colon = []243 after_colon = []244 colon = ""245 found_colon = False246 for i, source in enumerate(_XREF_REGEX.split(line)):247 if found_colon:248 after_colon.append(source)249 else:250 match = _SINGLE_COLON_REGEX.search(source)251 if (i % 2) == 0 and match:252 found_colon = True253 colon = source[match.start() : match.end()] # noqa: E203254 before_colon.append(source[: match.start()])255 after_colon.append(source[match.end() :]) # noqa: E203256 else:257 before_colon.append(source)...
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run lisa automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
