How to use tokenizer class

Best Atoum code snippet using tokenizer

ParserTest.php

Source:ParserTest.php

1<?php2/*3 * This file is part of Mustache.php.4 *5 * (c) 2010-2017 Justin Hileman6 *7 * For the full copyright and license information, please view the LICENSE8 * file that was distributed with this source code.9 */10/**11 * @group unit12 */13class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase14{15    /**16     * @dataProvider getTokenSets17     */18    public function testParse($tokens, $expected)19    {20        $parser = new Mustache_Parser();21        $this->assertEquals($expected, $parser->parse($tokens));22    }23    public function getTokenSets()24    {25        return array(26            array(27                array(),28                array(),29            ),30            array(31                array(array(32                    Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,33                    Mustache_Tokenizer::LINE  => 0,34                    Mustache_Tokenizer::VALUE => 'text',35                )),36                array(array(37                    Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,38                    Mustache_Tokenizer::LINE  => 0,39                    Mustache_Tokenizer::VALUE => 'text',40                )),41            ),42            array(43                array(array(44                    Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,45                    Mustache_Tokenizer::LINE => 0,46                    Mustache_Tokenizer::NAME => 'name',47                )),48                array(array(49                    Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,50                    Mustache_Tokenizer::LINE => 0,51                    Mustache_Tokenizer::NAME => 'name',52                )),53            ),54            array(55                array(56                    array(57                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,58                        Mustache_Tokenizer::LINE  => 0,59                        Mustache_Tokenizer::VALUE => 'foo',60                    ),61                    array(62                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,63                        Mustache_Tokenizer::LINE  => 0,64                        Mustache_Tokenizer::INDEX => 123,65                        Mustache_Tokenizer::NAME  => 'parent',66                    ),67                    array(68                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_ESCAPED,69                        Mustache_Tokenizer::LINE  => 0,70                        Mustache_Tokenizer::NAME  => 'name',71                    ),72                    array(73                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,74                        Mustache_Tokenizer::LINE  => 0,75                        Mustache_Tokenizer::INDEX => 456,76                        Mustache_Tokenizer::NAME  => 'parent',77                    ),78                    array(79                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,80                        Mustache_Tokenizer::LINE  => 0,81                        Mustache_Tokenizer::VALUE => 'bar',82                    ),83                ),84                array(85                    array(86                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,87                        Mustache_Tokenizer::LINE  => 0,88                        Mustache_Tokenizer::VALUE => 'foo',89                    ),90                    array(91                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,92                        Mustache_Tokenizer::NAME  => 'parent',93                        Mustache_Tokenizer::LINE  => 0,94                        Mustache_Tokenizer::INDEX => 123,95                        Mustache_Tokenizer::END   => 456,96                        Mustache_Tokenizer::NODES => array(97                            array(98                                Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,99                                Mustache_Tokenizer::LINE => 0,100                                Mustache_Tokenizer::NAME => 'name',101                            ),102                        ),103                    ),104                    array(105                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,106                        Mustache_Tokenizer::LINE  => 0,107                        Mustache_Tokenizer::VALUE => 'bar',108                    ),109                ),110            ),111            // This *would* be an invalid inheritance parse tree, but that pragma112            // isn't enabled so it'll thunk it back into an "escaped" token:113            array(114                array(115                    array(116                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,117                        Mustache_Tokenizer::NAME => 'foo',118                        Mustache_Tokenizer::OTAG => '{{',119                        Mustache_Tokenizer::CTAG => '}}',120                        Mustache_Tokenizer::LINE => 0,121                    ),122                    array(123                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,124                        Mustache_Tokenizer::LINE  => 0,125                        Mustache_Tokenizer::VALUE => 'bar',126                    ),127                ),128                array(129                    array(130                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,131                        Mustache_Tokenizer::NAME => '$foo',132                        Mustache_Tokenizer::OTAG => '{{',133                        Mustache_Tokenizer::CTAG => '}}',134                        Mustache_Tokenizer::LINE => 0,135                    ),136                    array(137                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,138                        Mustache_Tokenizer::LINE  => 0,139                        Mustache_Tokenizer::VALUE => 'bar',140                    ),141                ),142            ),143            array(144                array(145                    array(146                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,147                        Mustache_Tokenizer::LINE  => 0,148                        Mustache_Tokenizer::VALUE => '  ',149                    ),150                    array(151                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_DELIM_CHANGE,152                        Mustache_Tokenizer::LINE => 0,153                    ),154                    array(155                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,156                        Mustache_Tokenizer::LINE  => 0,157                        Mustache_Tokenizer::VALUE => "  \n",158                    ),159                    array(160                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,161                        Mustache_Tokenizer::NAME => 'foo',162                        Mustache_Tokenizer::OTAG => '[[',163                        Mustache_Tokenizer::CTAG => ']]',164                        Mustache_Tokenizer::LINE => 1,165                    ),166                ),167                array(168                    array(169                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,170                        Mustache_Tokenizer::NAME => 'foo',171                        Mustache_Tokenizer::OTAG => '[[',172                        Mustache_Tokenizer::CTAG => ']]',173                        Mustache_Tokenizer::LINE => 1,174                    ),175                ),176            ),177        );178    }179    /**180     * @dataProvider getInheritanceTokenSets181     */182    public function testParseWithInheritance($tokens, $expected)183    {184        $parser = new Mustache_Parser();185        $parser->setPragmas(array(Mustache_Engine::PRAGMA_BLOCKS));186        $this->assertEquals($expected, $parser->parse($tokens));187    }188    public function getInheritanceTokenSets()189    {190        return array(191            array(192                array(193                    array(194                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_PARENT,195                        Mustache_Tokenizer::NAME  => 'foo',196                        Mustache_Tokenizer::OTAG  => '{{',197                        Mustache_Tokenizer::CTAG  => '}}',198                        Mustache_Tokenizer::LINE  => 0,199                        Mustache_Tokenizer::INDEX => 8,200                    ),201                    array(202                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_BLOCK_VAR,203                        Mustache_Tokenizer::NAME  => 'bar',204                        Mustache_Tokenizer::OTAG  => '{{',205                        Mustache_Tokenizer::CTAG  => '}}',206                        Mustache_Tokenizer::LINE  => 0,207                        Mustache_Tokenizer::INDEX => 16,208                    ),209                    array(210                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,211                        Mustache_Tokenizer::LINE  => 0,212                        Mustache_Tokenizer::VALUE => 'baz',213                    ),214                    array(215                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,216                        Mustache_Tokenizer::NAME  => 'bar',217                        Mustache_Tokenizer::OTAG  => '{{',218                        Mustache_Tokenizer::CTAG  => '}}',219                        Mustache_Tokenizer::LINE  => 0,220                        Mustache_Tokenizer::INDEX => 19,221                    ),222                    array(223                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,224                        Mustache_Tokenizer::NAME  => 'foo',225                        Mustache_Tokenizer::OTAG  => '{{',226                        Mustache_Tokenizer::CTAG  => '}}',227                        Mustache_Tokenizer::LINE  => 0,228                        Mustache_Tokenizer::INDEX => 27,229                    ),230                ),231                array(232                    array(233                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_PARENT,234                        Mustache_Tokenizer::NAME  => 'foo',235                        Mustache_Tokenizer::OTAG  => '{{',236                        Mustache_Tokenizer::CTAG  => '}}',237                        Mustache_Tokenizer::LINE  => 0,238                        Mustache_Tokenizer::INDEX => 8,239                        Mustache_Tokenizer::END   => 27,240                        Mustache_Tokenizer::NODES => array(241                            array(242                                Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_BLOCK_ARG,243                                Mustache_Tokenizer::NAME  => 'bar',244                                Mustache_Tokenizer::OTAG  => '{{',245                                Mustache_Tokenizer::CTAG  => '}}',246                                Mustache_Tokenizer::LINE  => 0,247                                Mustache_Tokenizer::INDEX => 16,248                                Mustache_Tokenizer::END   => 19,249                                Mustache_Tokenizer::NODES => array(250                                    array(251                                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,252                                        Mustache_Tokenizer::LINE  => 0,253                                        Mustache_Tokenizer::VALUE => 'baz',254                                    ),255                                ),256                            ),257                        ),258                    ),259                ),260            ),261            array(262                array(263                    array(264                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,265                        Mustache_Tokenizer::NAME => 'foo',266                        Mustache_Tokenizer::OTAG => '{{',267                        Mustache_Tokenizer::CTAG => '}}',268                        Mustache_Tokenizer::LINE => 0,269                    ),270                    array(271                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,272                        Mustache_Tokenizer::LINE  => 0,273                        Mustache_Tokenizer::VALUE => 'bar',274                    ),275                    array(276                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,277                        Mustache_Tokenizer::NAME  => 'foo',278                        Mustache_Tokenizer::OTAG  => '{{',279                        Mustache_Tokenizer::CTAG  => '}}',280                        Mustache_Tokenizer::LINE  => 0,281                        Mustache_Tokenizer::INDEX => 11,282                    ),283                ),284                array(285                    array(286                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_BLOCK_VAR,287                        Mustache_Tokenizer::NAME  => 'foo',288                        Mustache_Tokenizer::OTAG  => '{{',289                        Mustache_Tokenizer::CTAG  => '}}',290                        Mustache_Tokenizer::LINE  => 0,291                        Mustache_Tokenizer::END   => 11,292                        Mustache_Tokenizer::NODES => array(293                            array(294                                Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,295                                Mustache_Tokenizer::LINE  => 0,296                                Mustache_Tokenizer::VALUE => 'bar',297                            ),298                        ),299                    ),300                ),301            ),302        );303    }304    /**305     * @dataProvider getBadParseTrees306     * @expectedException Mustache_Exception_SyntaxException307     */308    public function testParserThrowsExceptions($tokens)309    {310        $parser = new Mustache_Parser();311        $parser->parse($tokens);312    }313    public function getBadParseTrees()314    {315        return array(316            // no close317            array(318                array(319                    array(320                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,321                        Mustache_Tokenizer::NAME  => 'parent',322                        Mustache_Tokenizer::LINE  => 0,323                        Mustache_Tokenizer::INDEX => 123,324                    ),325                ),326            ),327            // no close inverted328            array(329                array(330                    array(331                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,332                        Mustache_Tokenizer::NAME  => 'parent',333                        Mustache_Tokenizer::LINE  => 0,334                        Mustache_Tokenizer::INDEX => 123,335                    ),336                ),337            ),338            // no opening inverted339            array(340                array(341                    array(342                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,343                        Mustache_Tokenizer::NAME  => 'parent',344                        Mustache_Tokenizer::LINE  => 0,345                        Mustache_Tokenizer::INDEX => 123,346                    ),347                ),348            ),349            // weird nesting350            array(351                array(352                    array(353                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,354                        Mustache_Tokenizer::NAME  => 'parent',355                        Mustache_Tokenizer::LINE  => 0,356                        Mustache_Tokenizer::INDEX => 123,357                    ),358                    array(359                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,360                        Mustache_Tokenizer::NAME  => 'child',361                        Mustache_Tokenizer::LINE  => 0,362                        Mustache_Tokenizer::INDEX => 123,363                    ),364                    array(365                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,366                        Mustache_Tokenizer::NAME  => 'parent',367                        Mustache_Tokenizer::LINE  => 0,368                        Mustache_Tokenizer::INDEX => 123,369                    ),370                    array(371                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,372                        Mustache_Tokenizer::NAME  => 'child',373                        Mustache_Tokenizer::LINE  => 0,374                        Mustache_Tokenizer::INDEX => 123,375                    ),376                ),377            ),378            // This *would* be a valid inheritance parse tree, but that pragma379            // isn't enabled here so it's going to fail :)380            array(381                array(382                    array(383                        Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_BLOCK_VAR,384                        Mustache_Tokenizer::NAME => 'foo',385                        Mustache_Tokenizer::OTAG => '{{',386                        Mustache_Tokenizer::CTAG => '}}',387                        Mustache_Tokenizer::LINE => 0,388                    ),389                    array(390                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,391                        Mustache_Tokenizer::LINE  => 0,392                        Mustache_Tokenizer::VALUE => 'bar',393                    ),394                    array(395                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,396                        Mustache_Tokenizer::NAME  => 'foo',397                        Mustache_Tokenizer::OTAG  => '{{',398                        Mustache_Tokenizer::CTAG  => '}}',399                        Mustache_Tokenizer::LINE  => 0,400                        Mustache_Tokenizer::INDEX => 11,401                    ),402                ),403            ),404        );405    }406}...

schema.php

Source:schema.php

...52	 * @throws NotSupportedException53	 */54	public function createFromString($str, $delimiter)55	{56		$tokenizer = Tokenizer::createFromString($str);57		foreach ($this->splitStatements($tokenizer, $delimiter) as $statement)58		{59			$this->executeStatement($statement);60		}61	}62	/**63	 * Splits tokens array into bunch of individual DDL statements.64	 *65	 * @param Tokenizer $tokenizer Tokens container.66	 * @param string $delimiter How to split DDL into statements.67	 *68	 * @return array[Tokenizer]69	 */70	protected function splitStatements(Tokenizer $tokenizer, $delimiter = ';')71	{72		$result = array();73		$index = 0;74		$result[$index] = array();75		/** @var Token $prevToken */76		$prevToken = null;77		/** @var Token $token */78		foreach ($tokenizer->getTokens() as $token)79		{80			if (81				$token->text === $delimiter82				&& $prevToken83				&& strpos($prevToken->text, "\n") !== false84			)85			{86				$index++;87				$result[$index] = array();88			}89			elseif (90				strpos($token->text , "\n") !== false91				&& $prevToken92				&& $prevToken->text === $delimiter93			)94			{95				array_pop($result[$index]);96				$index++;97				$result[$index] = array();98			}99			else100			{101				$result[$index][] = $token;102			}103			$prevToken = $token;104		}105		foreach ($result as $i => $tokens)106		{107			$result[$i] = Tokenizer::createFromTokens($tokens);108		}109		return $result;110	}111	/**112	 * Fills some schema part with information from one DDL statement.113	 *114	 * @param Tokenizer $tokenizer Statement tokens.115	 *116	 * @return void117	 * @throws NotSupportedException118	 */119	protected function executeStatement(Tokenizer $tokenizer)120	{121		/** @var Table $table */122		$tokenizer->resetState();123		$tokenizer->skipWhiteSpace();124		if ($tokenizer->testUpperText('CREATE'))125		{126			$this->executeCreate($tokenizer);127		}128		elseif ($tokenizer->testUpperText('INSERT'))129		{130			//skip insert into131		}132		elseif ($tokenizer->testUpperText('SET'))133		{134			//skip set identity_insert135		}136		elseif ($tokenizer->testUpperText('ALTER'))137		{138			$this->executeAlter($tokenizer);139		}140		elseif ($tokenizer->testUpperText('IF'))141		{142			$tokenizer->skipWhiteSpace();143			if ($tokenizer->testUpperText('OBJECT_ID'))144			{145				while (!$tokenizer->endOfInput())146				{147					if ($tokenizer->nextToken()->upper === 'CREATE')148						break;149				}150				$tokenizer->nextToken();151				$tokenizer->skipWhiteSpace();152				if ($tokenizer->testUpperText('TABLE'))153				{154					$this->executeCreateTable($tokenizer);155				}156				else157				{158					throw new NotSupportedException("'CREATE TABLE' expected. line:".$tokenizer->getCurrentToken()->line);159				}160			}161			elseif ($tokenizer->testUpperText('NOT'))162			{163				$tokenizer->skipWhiteSpace();164				if ($tokenizer->testUpperText('EXISTS'))165				{166					while (!$tokenizer->endOfInput())167					{168						if ($tokenizer->nextToken()->upper === 'CREATE')169							break;170					}171					$tokenizer->nextToken();172					$tokenizer->skipWhiteSpace();173					if ($tokenizer->testUpperText('UNIQUE'))174					{175						$unique = true;176						$tokenizer->skipWhiteSpace();177					}178					else179					{180						$unique = false;181					}182					if ($tokenizer->testUpperText('INDEX'))183					{184						$this->executeCreateIndex($tokenizer, $unique);185					}186					else187					{188						throw new NotSupportedException("'CREATE INDEX' expected. line:".$tokenizer->getCurrentToken()->line);189					}190				}191				else192				{193					throw new NotSupportedException("'NOT EXISTS' expected. line:".$tokenizer->getCurrentToken()->line);194				}195			}196			else197			{198				throw new NotSupportedException("'OBJECT_ID' expected. line:".$tokenizer->getCurrentToken()->line);199			}200		}201		elseif (!$tokenizer->endOfInput())202		{203			throw new NotSupportedException("'CREATE' expected. line:".$tokenizer->getCurrentToken()->line);204		}205	}206	/**207	 * @param Tokenizer $tokenizer Statement tokens.208	 *209	 * @return void210	 * @throws NotSupportedException211	 */212	protected function executeCreate(Tokenizer $tokenizer)213	{214		$tokenizer->skipWhiteSpace();215		if ($tokenizer->testUpperText("OR"))216		{217			$tokenizer->skipWhiteSpace();218			if ($tokenizer->testUpperText("REPLACE"))219				$tokenizer->skipWhiteSpace();220			else221				throw new NotSupportedException("'OR REPLACE' expected. line:".$tokenizer->getCurrentToken()->line);222		}223		if ($tokenizer->testUpperText('TABLE'))224		{225			$this->executeCreateTable($tokenizer);226		}227		elseif ($tokenizer->testUpperText('INDEX'))228		{229			$this->executeCreateIndex($tokenizer, false);230		}231		elseif ($tokenizer->testUpperText('UNIQUE'))232		{233			$tokenizer->skipWhiteSpace();234			if ($tokenizer->testUpperText('INDEX'))235				$tokenizer->skipWhiteSpace();236			$this->executeCreateIndex($tokenizer, true);237		}238		elseif ($tokenizer->testUpperText('TRIGGER'))239		{240			$this->executeCreateTrigger($tokenizer);241		}242		elseif (243			$tokenizer->testUpperText('PROCEDURE')244			|| $tokenizer->testUpperText('FUNCTION')245			|| $tokenizer->testUpperText('TYPE')246		)247		{248			$this->executeCreateProcedure($tokenizer);249		}250		elseif ($tokenizer->testUpperText('SEQUENCE'))251		{252			$this->executeCreateSequence($tokenizer);253		}254		else255		{256			throw new NotSupportedException("TABLE|INDEX|UNIQUE|TRIGGER|PROCEDURE|FUNCTION|TYPE|SEQUENCE expected. line:".$tokenizer->getCurrentToken()->line);257		}258	}259	/**260	 * @param Tokenizer $tokenizer Statement tokens.261	 *262	 * @return void263	 * @throws NotSupportedException264	 */265	protected function executeAlter(Tokenizer $tokenizer)266	{267		$tokenizer->skipWhiteSpace();268		if ($tokenizer->testUpperText('TABLE'))269		{270			$tokenizer->skipWhiteSpace();271			$tableName = $tokenizer->getCurrentToken()->text;272			/** @var Table $table */273			$table = $this->tables->search($tableName);274			if (!$table)275			{276				throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);277			}278			$tokenizer->nextToken();279			$tokenizer->skipWhiteSpace();280			if ($tokenizer->testUpperText('ADD'))281			{282				$tokenizer->skipWhiteSpace();283				if ($tokenizer->testUpperText('CONSTRAINT'))284				{285					$tokenizer->skipWhiteSpace();286					$table->createConstraint($tokenizer);287				}288			}289			elseif ($tokenizer->testUpperText('NOCHECK') || $tokenizer->testUpperText('CHECK'))290			{291				//(NOCHECK|CHECK) CONSTRAINT ALL292			}293			elseif ($tokenizer->testUpperText('DISABLE') || $tokenizer->testUpperText('ENABLE'))294			{295				//(DISABLE|ENABLE) TRIGGER ALL296			}297			else298			{299				throw new NotSupportedException("'ADD' expected. line:".$tokenizer->getCurrentToken()->line);300			}301		}302		else303		{304			throw new NotSupportedException("'TABLE' expected. line:".$tokenizer->getCurrentToken()->line);305		}306	}307	/**308	 * @param Tokenizer $tokenizer Statement tokens.309	 *310	 * @return void311	 * @throws NotSupportedException312	 */313	protected function executeCreateTable(Tokenizer $tokenizer)314	{315		$tokenizer->skipWhiteSpace();316		$this->tables->add(Table::create($tokenizer));317	}318	/**319	 * @param Tokenizer $tokenizer Statement tokens.320	 * @param boolean $unique Index uniqueness flag.321	 *322	 * @return void323	 * @throws NotSupportedException324	 */325	protected function executeCreateIndex(Tokenizer $tokenizer, $unique)326	{327		$tokenizer->skipWhiteSpace();328		$tokenizer->setBookmark();329		Index::searchTableName($tokenizer);330		$tableName = $tokenizer->getCurrentToken()->text;331		/** @var Table $table */332		$table = $this->tables->search($tableName);333		if (!$table)334		{335			throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);336		}337		$tokenizer->restoreBookmark();338		$table->createIndex($tokenizer, $unique);339	}340	/**341	 * @param Tokenizer $tokenizer Statement tokens.342	 *343	 * @return void344	 * @throws NotSupportedException345	 */346	protected function executeCreateTrigger(Tokenizer $tokenizer)347	{348		$tokenizer->skipWhiteSpace();349		$tokenizer->setBookmark();350		Trigger::searchTableName($tokenizer);351		$tableName = $tokenizer->getCurrentToken()->text;352		/** @var Table $table */353		$table = $this->tables->search($tableName);354		if (!$table)355		{356			throw new NotSupportedException("Table [$tableName] not found. line: ".$tokenizer->getCurrentToken()->line);357		}358		$tokenizer->restoreBookmark();359		$table->createTrigger($tokenizer);360	}361	/**362	 * @param Tokenizer $tokenizer Statement tokens.363	 *364	 * @return void365	 * @throws NotSupportedException366	 */367	protected function executeCreateProcedure(Tokenizer $tokenizer)368	{369		$tokenizer->putBack();370		$this->procedures->add(Procedure::create($tokenizer));371	}372	/**373	 * @param Tokenizer $tokenizer Statement tokens.374	 *375	 * @return void376	 * @throws NotSupportedException377	 */378	protected function executeCreateSequence(Tokenizer $tokenizer)379	{380		$tokenizer->skipWhiteSpace();381		$this->sequences->add(Sequence::create($tokenizer));382	}383}...

TokenizerTest.php

Source:TokenizerTest.php

...16     * @dataProvider getTokens17     */18    public function testScan($text, $delimiters, $expected)19    {20        $tokenizer = new Mustache_Tokenizer;21        $this->assertSame($expected, $tokenizer->scan($text, $delimiters));22    }23    public function getTokens()24    {25        return array(26            array(27                'text',28                null,29                array(30                    array(31                        Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,32                        Mustache_Tokenizer::LINE  => 0,33                        Mustache_Tokenizer::VALUE => 'text',34                    ),35                ),...

tokenizer

Using AI Code Generation

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\tests\units\test;3use \mageekguy\atoum\asserters;4use \mageekguy\atoum\exceptions;5use \mageekguy\atoum\tools\variable;6use \mageekguy\atoum\tools\diffs;7use \mageekguy\atoum\tools\asserters\phpString;8use \mageekguy\atoum\tools\asserters\phpArray;9use \mageekguy\atoum\tools\asserters\phpObject;10use \mageekguy\atoum\tools\asserters\phpFloat;11use \mageekguy\atoum\tools\asserters\phpInteger;12use \mageekguy\atoum\tools\asserters\phpBoolean;13use \mageekguy\atoum\tools\asserters\phpResource;14use \mageekguy\atoum\tools\asserters\phpGenerator;15use \mageekguy\atoum\tools\asserters\phpClass;16use \mageekguy\atoum\tools\asserters\phpFunction;17use \mageekguy\atoum\tools\asserters\phpConstant;18use \mageekguy\atoum\tools\asserters\phpCall;19use \mageekguy\atoum\tools\asserters\phpException;20use \mageekguy\atoum\tools\asserters\phpVariable;21use \mageekguy\atoum\tools\asserters\phpSizeOf;22use \mageekguy\atoum\tools\asserters\phpOutput;23use \mageekguy\atoum\tools\asserters\phpError;24use \mageekguy\atoum\tools\asserters\phpFile;25use \mageekguy\atoum\tools\asserters\phpString\utf8;26use \mageekguy\atoum\tools\asserters\phpString\binary;27use \mageekguy\atoum\tools\asserters\phpString\castToString;

tokenizer

Using AI Code Generation

1require_once 'vendor/autoload.php';2use atoum\atoum\report\fields\runner\tests\void\cli;3use atoum\atoum\report\fields\runner\tests\void\html;4use atoum\atoum\report\fields\runner\tests\void\text;5use atoum\atoum\report\fields\runner\tests\void\xunit;6use atoum\atoum\report\fields\runner\tests\void\yaml;7use atoum\atoum\report\fields\runner\tests\void\json;8use atoum\atoum\report\fields\runner\tests\void\csv;9use atoum\atoum\report\fields\runner\tests\void\cobertura;10use atoum\atoum\report\fields\runner\tests\void\sonarqube;11use atoum\atoum\report\fields\runner\tests\void\junit;12use atoum\atoum\report\fields\runner\failures\cli;13use atoum\atoum\report\fields\runner\failures\html;14use atoum\atoum\report\fields\runner\failures\text;15use atoum\atoum\report\fields\runner\failures\xunit;16use atoum\atoum\report\fields\runner\failures\yaml;17use atoum\atoum\report\fields\runner\failures\json;18use atoum\atoum\report\fields\runner\failures\csv;19use atoum\atoum\report\fields\runner\failures\cobertura;20use atoum\atoum\report\fields\runner\failures\sonarqube;21use atoum\atoum\report\fields\runner\failures\junit;22use atoum\atoum\report\fields\runner\outputs\cli;23use atoum\atoum\report\fields\runner\outputs\html;24use atoum\atoum\report\fields\runner\outputs\text;

tokenizer

Using AI Code Generation

1$tokenizer = new Tokenizer();2$tokens = $tokenizer->tokenize(file_get_contents('1.php'));3var_dump($tokens);4$tokens = token_get_all(file_get_contents('2.php'));5var_dump($tokens);6$tokenizer = new Tokenizer();7$tokens = $tokenizer->tokenize(file_get_contents('3.php'));8var_dump($tokens);9$tokenizer = new Tokenizer();10$tokens = $tokenizer->tokenize(file_get_contents('4.php'));11var_dump($tokens);12$tokenizer = new Tokenizer();13$tokens = $tokenizer->tokenize(file_get_contents('5.php'));14var_dump($tokens);15$tokenizer = new Tokenizer();16$tokens = $tokenizer->tokenize(file_get_contents('6.php'));17var_dump($tokens);18$tokenizer = new Tokenizer();19$tokens = $tokenizer->tokenize(file_get_contents('7.php'));20var_dump($tokens);21$tokenizer = new Tokenizer();22$tokens = $tokenizer->tokenize(file_get_contents('8.php'));23var_dump($tokens);24$tokenizer = new Tokenizer();25$tokens = $tokenizer->tokenize(file_get_contents('9.php'));26var_dump($tokens);27$tokenizer = new Tokenizer();28$tokens = $tokenizer->tokenize(file_get_contents('10.php'));29var_dump($tokens);30$tokenizer = new Tokenizer();31$tokens = $tokenizer->tokenize(file_get_contents('11.php'));32var_dump($tokens);33$tokenizer = new Tokenizer();

tokenizer

Using AI Code Generation

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\scripts\tokenizer;3$tokenizer = new tokenizer();4$tokenizer->addDirectory(__DIR__.'/src');5$tokenizer->run();6require_once 'vendor/autoload.php';7use \mageekguy\atoum\scripts\coverage;8$coverage = new coverage();9$coverage->addDirectory(__DIR__.'/src');10$coverage->run();11require_once 'vendor/autoload.php';12use \mageekguy\atoum\scripts\test;13$test = new test();14$test->addDirectory(__DIR__.'/src');15$test->run();16require_once 'vendor/autoload.php';17use \mageekguy\atoum\scripts\coverage;18$coverage = new coverage();19$coverage->addDirectory(__DIR__.'/src');20$coverage->run();21require_once 'vendor/autoload.php';22use \mageekguy\atoum\scripts\coverage;23$coverage = new coverage();24$coverage->addDirectory(__DIR__.'/src');25$coverage->run();26require_once 'vendor/autoload.php';27use \mageekguy\atoum\scripts\coverage;28$coverage = new coverage();29$coverage->addDirectory(__DIR__.'/src');30$coverage->run();31require_once 'vendor/autoload.php';32use \mageekguy\atoum\scripts\coverage;33$coverage = new coverage();34$coverage->addDirectory(__DIR__.'/src');35$coverage->run();36require_once 'vendor/autoload.php';37use \mageekguy\atoum\scripts\coverage;38$coverage = new coverage();39$coverage->addDirectory(__DIR__.'/src');

tokenizer

Using AI Code Generation

1$tokens = new atoum\atoum\cli\tokens();2$tokens->addToken('1', '1');3$tokens->addToken('2', '2');4$tokens->addToken('3', '3');5$tokens->addToken('4', '4');6$tokens->addToken('5', '5');7$tokens->addToken('6', '6');8$tokens->addToken('7', '7');9$tokens->addToken('8', '8');10$tokens->addToken('9', '9');11$tokens->addToken('plus', '+');12$tokens->addToken('minus', '-');13$tokens->addToken('multiply', '*');14$tokens->addToken('divide', '/');15$tokens->addToken('leftParenthesis', '(');16$tokens->addToken('rightParenthesis', ')');17$tokens->addToken('number', '[0-9]+', function($value) {return (int) $value;});18$tokens->addToken('whitespace', '\s+');19$tokens->addToken('end', '$');20$lexer = new atoum\atoum\cli\lexer($tokens);21$tokens = new atoum\atoum\cli\tokens();22$tokens->addToken('1', '1');23$tokens->addToken('2', '2');24$tokens->addToken('3', '3');25$tokens->addToken('4', '4');26$tokens->addToken('5', '5');27$tokens->addToken('6', '6');28$tokens->addToken('7', '7');29$tokens->addToken('8', '8');30$tokens->addToken('9', '9');31$tokens->addToken('plus', '+');32$tokens->addToken('minus', '-');33$tokens->addToken('multiply', '*');34$tokens->addToken('divide', '/');35$tokens->addToken('leftParenthesis', '(');36$tokens->addToken('rightParenthesis', ')');37$tokens->addToken('number', '[0-9]+', function($value) {return (int) $value;});38$tokens->addToken('whitespace', '\s+');39$tokens->addToken('end', '$');

tokenizer

Using AI Code Generation

1require_once 'vendor/autoload.php';2use \mageekguy\atoum\asserters;3$php = new \mageekguy\atoum\php\tokenizer();4$php->parseFile('2.php');5$parser = (new PhpParser\ParserFactory)->create(PhpParser\ParserFactory::PREFER_PHP7);6try {7    $stmts = $parser->parse(file_get_contents('2.php'));8} catch (PhpParser\Error $e) {9    echo 'Parse Error: ', $e->getMessage();10}11$traverser = new PhpParser\NodeTraverser;12$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);13$traverser->addVisitor(new MyNodeVisitor);14$ast = $traverser->traverse($stmts);15$prettyPrinter = new PhpParser\PrettyPrinter\Standard;16echo $prettyPrinter->prettyPrintFile($ast);17$traverser = new PhpParser\NodeTraverser;18$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);19$traverser->addVisitor(new MyNodeVisitor2);20$ast = $traverser->traverse($stmts);21$prettyPrinter = new PhpParser\PrettyPrinter\Standard;22echo $prettyPrinter->prettyPrintFile($ast);23$traverser = new PhpParser\NodeTraverser;24$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);25$traverser->addVisitor(new MyNodeVisitor3);26$ast = $traverser->traverse($stmts);27$prettyPrinter = new PhpParser\PrettyPrinter\Standard;28echo $prettyPrinter->prettyPrintFile($ast);29$traverser = new PhpParser\NodeTraverser;30$traverser->addVisitor(new PhpParser\NodeVisitor\NameResolver);31$traverser->addVisitor(new MyNodeVisitor4);32$ast = $traverser->traverse($stmts);33$prettyPrinter = new PhpParser\PrettyPrinter\Standard;

tokenizer

Using AI Code Generation

1$tokenizer = new \mageekguy\atoum\tokenizer();2$tokens = $tokenizer->parseFile('2.php');3$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');4$tokenizer = new \mageekguy\atoum\tokenizer();5$tokens = $tokenizer->parseFile('2.php');6$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');7$tokenizer = new \mageekguy\atoum\tokenizer();8$tokens = $tokenizer->parseFile('3.php');9$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');10$tokenizer = new \mageekguy\atoum\tokenizer();11$tokens = $tokenizer->parseFile('4.php');12$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');13$tokenizer = new \mageekguy\atoum\tokenizer();14$tokens = $tokenizer->parseFile('5.php');15$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');16$tokenizer = new \mageekguy\atoum\tokenizer();17$tokens = $tokenizer->parseFile('6.php');18$tokens = $tokenizer->parseString('<?php echo "Hello World"; ?>');19$tokenizer = new \mageekguy\atoum\tokenizer();20$tokens = $tokenizer->parseFile('7.php');21$tokens = $tokenizer->parseString('<?php echo "Hello World

tokenizer

Using AI Code Generation

1$tokenizer = new atoum\php\tokenizer(file_get_contents('1.php'));2$tokenizer->parse();3$tokens = $tokenizer->getTokens();4$tokens->rewind();5while ($tokens->valid()) {6    $token = $tokens->current();7    if ($token[0] == T_STRING) {8";9    }10    $tokens->next();11}12$tokenizer = new atoum\php\tokenizer(file_get_contents('2.php'));13$tokenizer->parse();14$tokens = $tokenizer->getTokens();15$tokens->rewind();16while ($tokens->valid()) {17    $token = $tokens->current();18    if ($token[0] == T_STRING) {19";20    }21    $tokens->next();22}23$tokenizer = new atoum\php\tokenizer(file_get_contents('3.php'));24$tokenizer->parse();25$tokens = $tokenizer->getTokens();26$tokens->rewind();27while ($tokens->valid()) {28    $token = $tokens->current();29    if ($token[0] == T_STRING) {30";31    }32    $tokens->next();33}34$tokenizer = new atoum\php\tokenizer(file_get_contents('4.php'));35$tokenizer->parse();36$tokens = $tokenizer->getTokens();37$tokens->rewind();38while ($tokens->valid()) {39    $token = $tokens->current();40    if ($token[0] == T_STRING) {41";42    }43    $tokens->next();44}45$tokenizer = new atoum\php\tokenizer(file_get_contents('5.php'));46$tokenizer->parse();

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.