Test your AI Agents with the all-new Agent to Agent Testing Platform.Learn More

How to use _get_token method in Slash

Best Python code snippet using slash

tokens.py

Source:tokens.py

...133        return self._index >= self.source.size134    def _new_token(self):135        self._token_start = self._index136        self._token_text = ""137    def _get_token(self, type: TokenType) -> Token:138        return Token(type, self._token_text, SourceLocation(self._token_start, len(self._token_text), self.source))139    def iter_tokens(self) -> Iterator[Token]:140        while not self.is_done():141            if self._curr == "\n":142                self._advance()143            elif self._curr in " \t\r":144                self._advance()145            elif self._curr == "#":146                while self._curr != "\n":147                    self._advance()148            elif self._curr == "+":149                self._new_token()150                if self._next == "=":151                    self._advance(steps=2)152                    yield self._get_token(TokenType.PLUS_EQUAL)153                else:154                    self._advance()155                    yield self._get_token(TokenType.PLUS)156            elif self._curr == "-":157                self._new_token()158                if self._next == "=":159                    self._advance(steps=2)160                    yield self._get_token(TokenType.MINUS_EQUAL)161                elif self._next == ">":162                    self._advance(steps=2)163                    yield self._get_token(TokenType.ARROW)164                else:165                    self._advance()166                    yield self._get_token(TokenType.MINUS)167            elif self._curr == "*":168                self._new_token()169                self._advance()170                if self._curr == "=":171                    self._advance()172                    yield self._get_token(TokenType.STAR_EQUAL)173                # elif self._next == "*":174                #     self._advance()175                #     if self._curr == "=":176                #         self._advance()177                #         yield self._get_token(TokenType.STAR_STAR_EQUAL)178                #     else:179                #         yield self._get_token(TokenType.STAR_STAR)180                else:181                    yield self._get_token(TokenType.STAR)182            elif self._curr == "/":183                self._new_token()184                self._advance()185                if self._curr == "=":186                    self._advance()187                    yield self._get_token(TokenType.SLASH_EQUAL)188                # elif self._curr == "/":189                #     self._advance()190                #     if self._curr == "=":191                #         self._advance()192                #         yield self._get_token(TokenType.SLASH_SLASH_EQUAL)193                #     else:194                #         yield self._get_token(TokenType.SLASH_SLASH)195                else:196                    yield self._get_token(TokenType.SLASH)197            elif self._curr == "%":198                self._new_token()199                if self._next == "=":200                    self._advance(steps=2)201                    yield self._get_token(TokenType.PERCENT_EQUAL)202                else:203                    self._advance()204                    yield self._get_token(TokenType.PERCENT)205            elif self._curr == "<":206                self._new_token()207                self._advance()208                if self._curr == "=":209                    self._advance()210                    yield self._get_token(TokenType.LESS_EQUAL)211                else:212                    yield self._get_token(TokenType.LESS)213            elif self._curr == ">":214                self._new_token()215                self._advance()216                if self._curr == "=":217                    self._advance()218                    yield self._get_token(TokenType.GREATER_EQUAL)219                else:220                    yield self._get_token(TokenType.GREATER)221            elif self._curr == "=":222                self._new_token()223                if self._next == "=":224                    self._advance(steps=2)225                    yield self._get_token(TokenType.EQUAL_EQUAL)226                else:227                    self._advance()228                    yield self._get_token(TokenType.EQUAL)229            elif self._curr == ":":230                self._new_token()231                if self._next == ":":232                    self._advance(steps=2)233                    yield self._get_token(TokenType.COLON_COLON)234                else:235                    self._advance()236                    yield self._get_token(TokenType.COLON)237            elif self._curr == "!" and self._next == "=":238                self._new_token()239                self._advance(steps=2)240                yield self._get_token(TokenType.NOT_EQUAL)241            elif self._curr in SIMPLE_TOKENS.keys():242                type = SIMPLE_TOKENS[self._curr]243                self._new_token()244                self._advance()245                yield self._get_token(type)246            elif self._curr == "\"":247                self._new_token()248                self._advance()249                while self._curr != "\"":250                    if self._curr == "\n":251                        raise CompilerMessage(ErrorType.PARSE, "A string literal cannot span multiple lines", self._get_token(TokenType.ERROR).location)252                    if self._curr == "\\":253                        if self._next == "\n":254                            raise CompilerMessage(ErrorType.PARSE, "A string literal cannot span multiple lines", self._get_token(TokenType.ERROR).location)255                        self._advance()256                    self._advance()257                self._advance()258                yield self._get_token(TokenType.STRING)259            elif self._curr.isalpha() or self._curr == "_":260                self._new_token()261                while self._curr.isalpha() or self._curr.isnumeric() or self._curr == "_":262                    self._advance()263                tok = self._get_token(TokenType.IDENT)264                if tok.text in KEYWORDS:265                    yield Token(KEYWORDS[tok.text], tok.text, tok.location)266                else:267                    yield tok268            elif self._curr.isnumeric():269                self._new_token()270                if self._curr == "0" and self._next in "bxBX":271                    self._advance(steps=2)272                    if self._last in "bB":273                        while self._curr.isnumeric():274                            if self._curr not in "01":275                                self._advance()276                                raise CompilerMessage(ErrorType.PARSE, "Digits other than 0 and 1 are not allowed in a binary literal.",277                                                      self._get_token(TokenType.ERROR).location)278                            else:279                                self._advance()280                        yield self._get_token(TokenType.BINARY)281                    else:  # self.last in "xX"282                        while self._curr.isnumeric() or self._curr in "abcdefABCDEF":283                            self._advance()284                        yield self._get_token(TokenType.HEX)285                else:286                    self._new_token()287                    while self._curr.isnumeric():288                        self._advance()289                    if self._curr == "." and self._next.isnumeric():290                        self._advance()291                        while self._curr.isnumeric():292                            self._advance()293                        yield self._get_token(TokenType.FLOAT)294                    else:295                        yield self._get_token(TokenType.INTEGER)296            else:297                self._new_token()298                self._advance()299                raise CompilerMessage(ErrorType.PARSE, f"Unexpected character `{self._last}`.", self._get_token(TokenType.ERROR).location)300        self._new_token()301        while True:...

pyconfig.py

Source:pyconfig.py

...117        """118        Parse config file and store results to this object.119        """120        while True:121            self._get_token()122            if self._tok_value == "from":123                self._get_from_import()124                continue125            if keyword.iskeyword(self._tok_value):126                raise SyntaxError("Cannot assign to a python keyword: %s" % self._tok_value)127            if self._tok_name == "ENDMARKER":128                break129            self._assert_token(("NAME", ))130            key = self._tok_value131            self._get_token()132            self._assert_token(("OP", "="))133            value = self._get_value()134            yield key, value135    def _assert_token(self, *args):136        """137        Check if token has proper name and value.138        *args are tuples (name, value), (name, )139        """140        for i in args:141            if len(i) == 1 and i == (self._tok_name, ):142                return143            if len(i) == 2 and i == (self._tok_name, self._tok_value):144                return145        raise SyntaxError("Invalid syntax: file: %s, begin: %s, end: %s, text: %s" % (self._open_file, self._tok_begin, self._tok_end, self._tok_line))146    def _get_token(self, skip_newline=True):147        """148        Get a new token from token generator.149        """150        self._tok_number, self._tok_value, self._tok_begin, self._tok_end, self._tok_line = next(self._tokens)151        self._tok_name = token.tok_name.get(self._tok_number, None)152        if self._debug:153            print("%2s %16s %s" % (self._tok_number, self._tok_name, self._tok_value.strip()))154        # skip some tokens155        if self._tok_name in ["COMMENT", "INDENT", "DEDENT"]:156            self._get_token()157        if skip_newline and self._tok_name in ["NL", "NEWLINE"]:158            self._get_token()159    def _get_NAME(self):160        """161        Return a NAME token value.162        """163        if self._tok_value == "False":164            return False165        if self._tok_value == "True":166            return True167        if self._tok_value == "None":168            return None169        # return already defined variable170        return self[self._tok_value]171    def _get_STRING(self):172        """173        Return a STRING token value.174        """175        # remove apostrophes or quotation marks176        result = self._tok_value[1:-1]177        # look at next token if "%s" follows the string178        self._tokens, tmp = itertools.tee(self._tokens)179        if next(tmp)[1:2] != ("%", ):180            # just a regular string181            return result182        # string formatting is used183        self._get_token()184        self._assert_token(("OP", "%"))185        values = self._get_value()186        return result % values187    def _get_NUMBER(self, negative=False):188        """189        Return a NUMBER token value.190        """191        if self._tok_value.find(".") != -1:192            result = float(self._tok_value)193        else:194            result = int(self._tok_value)195        if negative:196            return -result197        return result198    def _get_value(self, get_next=True, basic_types_only=False):199        """200        Get a value (number, string, other variable value, ...).201        """202        if get_next:203            self._get_token()204        self._assert_token(("NAME", ), ("NUMBER", ), ("STRING", ), ("OP", "{"), ("OP", "["), ("OP", "("), ("OP", "-"))205        if (self._tok_name, self._tok_value) == ("OP", "-"):206            self._get_token()207            self._assert_token(("NUMBER", ))208            return self._get_NUMBER(negative=True)209        if self._tok_name in ["NAME", "NUMBER", "STRING"]:210            return getattr(self, "_get_%s" % self._tok_name)()211        if not basic_types_only:212            if (self._tok_name, self._tok_value) == ("OP", "{"):213                return self._get_dict()214            if (self._tok_name, self._tok_value) == ("OP", "["):215                return self._get_list()216            if (self._tok_name, self._tok_value) == ("OP", "("):217                return self._get_tuple()218        self._assert_token(("FOO", ))219    def _get_from_import(self):220        """221        Parse 'from <config> import <variables/*>'222        and import <config> data to this object.223        """224        file_name = ""225        while True:226            self._get_token()227            if (self._tok_name, self._tok_value) == ("NAME", "import"):228                break229            file_name += str(self._tok_value)230        file_name += "." + self._config_file_suffix231        file_name = os.path.join(os.path.dirname(self._open_file), file_name)232        self._assert_token(("NAME", "import"))233        imports = []234        self._get_token()235        while self._tok_name not in ("NL", "NEWLINE"):236            self._assert_token(("NAME", ), ("OP", "*"))237            imports.append(self._tok_value)238            self._get_token(skip_newline=False)239            self._skip_commas(skip_newline=False)240        imported_config = self.__class__(config_file_suffix=self._config_file_suffix, debug=self._debug)241        imported_config.load_from_file(file_name)242        if "*" in imports:243            self.load_from_dict(imported_config)244        else:245            for key in imports:246                try:247                    self[key] = imported_config[key]248                except KeyError:249                    raise KeyError("Can't import %s from %s." % (key, file_name))250    def _skip_commas(self, skip_newline=True):251        """252        Skip OP tokens which contain commas.253        """254        while (self._tok_name, self._tok_value) == ("OP", ","):255            self._get_token(skip_newline)256    def _get_dict(self):257        """258        Get a dictionary content.259        """260        result = {}261        while True:262            self._get_token()263            self._skip_commas()264            if (self._tok_name, self._tok_value) == ("OP", "}"):265                break266            key = self._get_value(get_next=False, basic_types_only=True)267            self._get_token()268            self._assert_token(("OP", ":"))269            value = self._get_value()270            result[key] = value271        return result272    def _get_list(self):273        """274        Get a list content.275        """276        result = []277        while True:278            self._get_token()279            self._skip_commas()280            if (self._tok_name, self._tok_value) == ("OP", "]"):281                break282            value = self._get_value(get_next=False)283            result.append(value)284        return result285    def _get_tuple(self):286        """287        Get a tuple content.288        """289        result = []290        while True:291            self._get_token()292            self._skip_commas()293            if (self._tok_name, self._tok_value) == ("OP", ")"):294                break295            value = self._get_value(get_next=False)296            result.append(value)297        return tuple(result)298# settings is created only if PROJECT_CONFIG_FILE is set299if "PROJECT_CONFIG_FILE" in os.environ:300    try:301        settings = PyConfigParser()302        settings.load_from_file(os.environ.get("PROJECT_CONFIG_FILE"))303    except Exception as ex:...

cleaner.py

Source:cleaner.py

...22        """23        clean_text = ""24        tokens = self._tokenizer.tokenize(self._text)25        i = 026        token = self._get_token(tokens, i)27        while token is not None:28            value = token.value29            tag = token.tag30            if tag == TokenTagEnum.WORD or tag == TokenTagEnum.NUMBER:31                if "'" in value:32                    index = value.find("'")33                    value = value[0:index]34                forward_token = self._get_token(tokens, i + 1)35                if value != "":36                    if forward_token is not None and forward_token.value.startswith("'"):37                        clean_text += value38                    else:39                        clean_text += value + " "40            elif tag == TokenTagEnum.URL:41                clean_text += value42            elif value == "(":  # NOTE: skip parenthesis (...)43                j = i + 144                forward_token = self._get_token(tokens, j)45                while forward_token is not None and forward_token.value != ")":46                    j += 147                    forward_token = self._get_token(tokens, j)48                i = j49            elif value == "[":  # NOTE: skip parenthesis [...]50                j = i + 151                forward_token = self._get_token(tokens, j)52                while forward_token is not None and forward_token.value != "]":53                    j += 154                    forward_token = self._get_token(tokens, j)55                i = j56            elif value == "'":57                if len(clean_text) > 0 and clean_text[-1] == " ":58                    clean_text = clean_text[0:-1]59                    clean_text += value60            i += 161            token = self._get_token(tokens, i)62        """63        clean_text = self._text64        clean_text = clean_text.strip().lower()65        clean_text = clean_text.replace(".", " ")66        clean_text = clean_text.replace("-", " ")67        clean_text = clean_text.replace("_", " ")68        clean_text = clean_text.replace("/", " ")69        clean_text = re.sub(r'\([^)]*\)', '', clean_text)   # Remove (<any>)70        clean_text = re.sub(r'\[[^)]*\]', '', clean_text)   # Remove [<any>]71        clean_text = re.sub(r'\{[^)]*\}', '', clean_text)   # Remove {<any>}72        clean_text = nlp.remove_punctuations(clean_text, string.punctuation.replace("'", ""))73        clean_text = nlp.remove_extra_spaces(clean_text)74        return clean_text75    def _get_token(self, tokens, idx):76        if len(tokens) <= idx:77            return None78        return tokens[idx]79    def _is_relevant_word(self, word):...

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.