How to use tokenise method in wpt
Best JavaScript code snippet using wpt
tokenise.test.ts
Source:tokenise.test.ts 
...119/* Terms */120/***************/121function atom1() {122 let expected = [new Atom("good_atom")];123 let [actual, _] = ErlangJs.tokenise("good_atom");124 assert.deepStrictEqual(actual, expected);125}126function atom2() {127 let expected = [new Atom("good atom")];128 let [actual, _] = ErlangJs.tokenise("'good atom'");129 assert.deepStrictEqual(actual, expected);130}131function variable1() {132 let expected = [new Variable("GOOD_VAR")];133 let [actual, _] = ErlangJs.tokenise("GOOD_VAR");134 assert.deepStrictEqual(actual, expected);135}136function variable2() {137 let expected = [new Variable("_GOOD_VAR")];138 let [actual, _] = ErlangJs.tokenise("_GOOD_VAR");139 assert.deepStrictEqual(actual, expected);140}141function integer1() {142 let expected = [new Integer(1)];143 let [actual, _] = ErlangJs.tokenise("1");144 assert.deepStrictEqual(actual, expected);145}146function float1() {147 let expected = [new Float(1.0)];148 let [actual, _] = ErlangJs.tokenise("1.0");149 assert.deepStrictEqual(actual, expected);150}151function float2() {152 let expected = [new Float(1.1)];153 let [actual, _] = ErlangJs.tokenise("1.1");154 assert.deepStrictEqual(actual, expected);155}156function tuple1() {157 let expected = [new Tuple()];158 let [actual, _] = ErlangJs.tokenise("{}");159 assert.deepStrictEqual(actual, expected);160}161function tuple2() {162 let expected = [new Tuple(new Atom("atom"))];163 let [actual, _] = ErlangJs.tokenise("{atom}");164 assert.deepStrictEqual(actual, expected);165}166function tuple3() {167 let expected = [new Tuple(new Atom("atom1"), new Atom("atom2"))];168 let [actual, _] = ErlangJs.tokenise("{atom1, atom2}");169 assert.deepStrictEqual(actual, expected);170}171function tuple4() {172 let expected = [new Tuple(new Atom("atom1"), new Tuple(new Atom("atom2")))];173 let [actual, _] = ErlangJs.tokenise("{atom1, {atom2}}");174 assert.deepStrictEqual(actual, expected);175}176function list1() {177 let expected = [new List()];178 let [actual, _] = ErlangJs.tokenise("[]");179 assert.deepStrictEqual(actual, expected);180}181function list2() {182 let expected = [new List(new Atom("atom"))];183 let [actual, _] = ErlangJs.tokenise("[atom]");184 assert.deepStrictEqual(actual, expected);185}186function list3() {187 let expected = [new List(new Atom("atom1"), new Atom("atom2"))];188 let [actual, _] = ErlangJs.tokenise("[atom1, atom2]");189 assert.deepStrictEqual(actual, expected);190}191function list4() {192 let expected = [new List(new Atom("atom1"), new List(new Atom("atom2")))];193 let [actual, _] = ErlangJs.tokenise("[atom1, [atom2]]");194 assert.deepStrictEqual(actual, expected);195}196function parenthesis1() {197 let expected = [new Parenthesis()];198 let [actual, _] = ErlangJs.tokenise("()");199 assert.deepStrictEqual(actual, expected);200}201function parenthesis2() {202 let expected = [new Parenthesis(new Atom("atom"))];203 let [actual, _] = ErlangJs.tokenise("(atom)");204 assert.deepStrictEqual(actual, expected);205}206function parenthesis3() {207 let expected = [new Parenthesis(new Atom("atom1"), new Atom("atom2"))];208 let [actual, _] = ErlangJs.tokenise("(atom1, atom2)");209 assert.deepStrictEqual(actual, expected);210}211function parenthesis4() {212 let expected = [new Parenthesis(new Atom("atom1"), new Parenthesis(new Atom("atom2")))];213 let [actual, _] = ErlangJs.tokenise("(atom1, (atom2))");214 assert.deepStrictEqual(actual, expected);215}216/*************/217/* Operators */218/*************/219function setter1() {220 let expected = [new Setter(new Variable("Var"), new Atom("atom"))];221 let [actual, _] = ErlangJs.tokenise("Var = atom");222 assert.deepStrictEqual(actual, expected);223}224function setter2() {225 let expected = [new Setter(new Atom("atom"), new Atom("atom"))];226 let [actual, _] = ErlangJs.tokenise("atom = atom");227 assert.deepStrictEqual(actual, expected);228}229function setter3() {230 let expected = [new Setter(null, new Atom("atom"))];231 let [actual, _] = ErlangJs.tokenise(" = atom");232 assert.deepStrictEqual(actual, expected);233}234function setter4() {235 let expected = [new Setter(new Variable("Var"), null)];236 let [actual, _] = ErlangJs.tokenise("Var = ");237 assert.deepStrictEqual(actual, expected);238}239function append1() {240 let expected = [new Append(new Variable("Var"), new Atom("atom"))];241 let [actual, _] = ErlangJs.tokenise("Var ++ atom");242 assert.deepStrictEqual(actual, expected);243}244function append2() {245 let expected = [new Append(new Atom("atom"), new Atom("atom"))];246 let [actual, _] = ErlangJs.tokenise("atom ++ atom");247 assert.deepStrictEqual(actual, expected);248}249function append3() {250 let expected = [new Append(null, new Atom("atom"))];251 let [actual, _] = ErlangJs.tokenise(" ++ atom");252 assert.deepStrictEqual(actual, expected);253}254function append4() {255 let expected = [new Append(new Variable("Var"), null)];256 let [actual, _] = ErlangJs.tokenise("Var ++ ");257 assert.deepStrictEqual(actual, expected);258}259function subtract1() {260 let expected = [new Subtract(new Variable("Var"), new Atom("atom"))];261 let [actual, _] = ErlangJs.tokenise("Var -- atom");262 assert.deepStrictEqual(actual, expected);263}264function subtract2() {265 let expected = [new Subtract(new Atom("atom"), new Atom("atom"))];266 let [actual, _] = ErlangJs.tokenise("atom -- atom");267 assert.deepStrictEqual(actual, expected);268}269function subtract3() {270 let expected = [new Subtract(null, new Atom("atom"))];271 let [actual, _] = ErlangJs.tokenise(" -- atom");272 assert.deepStrictEqual(actual, expected);273}274function subtract4() {275 let expected = [new Subtract(new Variable("Var"), null)];276 let [actual, _] = ErlangJs.tokenise("Var -- ");277 assert.deepStrictEqual(actual, expected);278}279function add1() {280 let expected = [new Add(new Variable("Var"), new Atom("atom"))];281 let [actual, _] = ErlangJs.tokenise("Var + atom");282 assert.deepStrictEqual(actual, expected);283}284function add2() {285 let expected = [new Add(new Atom("atom"), new Atom("atom"))];286 let [actual, _] = ErlangJs.tokenise("atom + atom");287 assert.deepStrictEqual(actual, expected);288}289function add3() {290 let expected = [new Add(null, new Atom("atom"))];291 let [actual, _] = ErlangJs.tokenise(" + atom");292 assert.deepStrictEqual(actual, expected);293}294function add4() {295 let expected = [new Add(new Variable("Var"), null)];296 let [actual, _] = ErlangJs.tokenise("Var + ");297 assert.deepStrictEqual(actual, expected);298}299function minus1() {300 let expected = [new Minus(new Variable("Var"), new Atom("atom"))];301 let [actual, _] = ErlangJs.tokenise("Var - atom");302 assert.deepStrictEqual(actual, expected);303}304function minus2() {305 let expected = [new Minus(new Atom("atom"), new Atom("atom"))];306 let [actual, _] = ErlangJs.tokenise("atom - atom");307 assert.deepStrictEqual(actual, expected);308}309function minus3() {310 let expected = [new Minus(null, new Atom("atom"))];311 let [actual, _] = ErlangJs.tokenise(" - atom");312 assert.deepStrictEqual(actual, expected);313}314function minus4() {315 let expected = [new Minus(new Variable("Var"), null)];316 let [actual, _] = ErlangJs.tokenise("Var - ");317 assert.deepStrictEqual(actual, expected);318}319function multiply1() {320 let expected = [new Multiply(new Variable("Var"), new Atom("atom"))];321 let [actual, _] = ErlangJs.tokenise("Var * atom");322 assert.deepStrictEqual(actual, expected);323}324function multiply2() {325 let expected = [new Multiply(new Atom("atom"), new Atom("atom"))];326 let [actual, _] = ErlangJs.tokenise("atom * atom");327 assert.deepStrictEqual(actual, expected);328}329function multiply3() {330 let expected = [new Multiply(null, new Atom("atom"))];331 let [actual, _] = ErlangJs.tokenise(" * atom");332 assert.deepStrictEqual(actual, expected);333}334function multiply4() {335 let expected = [new Multiply(new Variable("Var"), null)];336 let [actual, _] = ErlangJs.tokenise("Var * ");337 assert.deepStrictEqual(actual, expected);338}339function divide1() {340 let expected = [new Divide(new Variable("Var"), new Atom("atom"))];341 let [actual, _] = ErlangJs.tokenise("Var / atom");342 assert.deepStrictEqual(actual, expected);343}344function divide2() {345 let expected = [new Divide(new Atom("atom"), new Atom("atom"))];346 let [actual, _] = ErlangJs.tokenise("atom / atom");347 assert.deepStrictEqual(actual, expected);348}349function divide3() {350 let expected = [new Divide(null, new Atom("atom"))];351 let [actual, _] = ErlangJs.tokenise(" / atom");352 assert.deepStrictEqual(actual, expected);353}354function divide4() {355 let expected = [new Divide(new Variable("Var"), null)];356 let [actual, _] = ErlangJs.tokenise("Var / ");357 assert.deepStrictEqual(actual, expected);358}359function compare1(testObject: TestSuite.Object) {360 let expected = [new Compare(new Variable("Var"), new Atom("atom"))];361 let [actual, _] = ErlangJs.tokenise("Var == atom");362 testObject.log(expected, actual);363 assert.deepStrictEqual(actual, expected);364}365function compare2(testObject: TestSuite.Object) {366 let expected = [new Compare(new Atom("atom"), new Atom("atom"))];367 let [actual, _] = ErlangJs.tokenise("atom == atom");368 testObject.log(expected, actual);369 assert.deepStrictEqual(actual, expected);370}371function compare3(testObject: TestSuite.Object) {372 let expected = [new Compare(null, new Atom("atom"))];373 let [actual, _] = ErlangJs.tokenise(" == atom");374 testObject.log(expected, actual);375 assert.deepStrictEqual(actual, expected);376}377function compare4(testObject: TestSuite.Object) {378 let expected = [new Compare(new Variable("Var"), null)];379 let [actual, _] = ErlangJs.tokenise("Var == ");380 testObject.log(expected, actual);381 assert.deepStrictEqual(actual, expected);382}383function compare5(testObject: TestSuite.Object) {384 let expected = [new Compare(new Variable("Var"), new Atom("atom"), {not: true})];385 let [actual, _] = ErlangJs.tokenise("Var /= atom");386 testObject.log(expected, actual);387 assert.deepStrictEqual(actual, expected);388}389function compare6(testObject: TestSuite.Object) {390 let expected = [new Compare(new Atom("atom"), new Atom("atom"), {not: true})];391 let [actual, _] = ErlangJs.tokenise("atom /= atom");392 testObject.log(expected, actual);393 assert.deepStrictEqual(actual, expected);394}395function compare7(testObject: TestSuite.Object) {396 let expected = [new Compare(null, new Atom("atom"), {not: true})];397 let [actual, _] = ErlangJs.tokenise(" /= atom");398 testObject.log(expected, actual);399 assert.deepStrictEqual(actual, expected);400}401function compare8(testObject: TestSuite.Object) {402 let expected = [new Compare(new Variable("Var"), null, {not: true})];403 let [actual, _] = ErlangJs.tokenise("Var /= ");404 testObject.log(expected, actual);405 assert.deepStrictEqual(actual, expected);406}407function compare9(testObject: TestSuite.Object) {408 let expected = [new Compare(new Variable("Var"), new Atom("atom"), {exact: true})];409 let [actual, _] = ErlangJs.tokenise("Var =:= atom");410 testObject.log(expected, actual);411 assert.deepStrictEqual(actual, expected);412}413function compare10(testObject: TestSuite.Object) {414 let expected = [new Compare(new Atom("atom"), new Atom("atom"), {exact: true})];415 let [actual, _] = ErlangJs.tokenise("atom =:= atom");416 testObject.log(expected, actual);417 assert.deepStrictEqual(actual, expected);418}419function compare11(testObject: TestSuite.Object) {420 let expected = [new Compare(null, new Atom("atom"), {exact: true})];421 let [actual, _] = ErlangJs.tokenise(" =:= atom");422 testObject.log(expected, actual);423 assert.deepStrictEqual(actual, expected);424}425function compare12(testObject: TestSuite.Object) {426 let expected = [new Compare(new Variable("Var"), null, {exact: true})];427 let [actual, _] = ErlangJs.tokenise("Var =:= ");428 testObject.log(expected, actual);429 assert.deepStrictEqual(actual, expected);430}431function compare13(testObject: TestSuite.Object) {432 let expected = [new Compare(new Variable("Var"), new Atom("atom"), {exact: true, not: true})];433 let [actual, _] = ErlangJs.tokenise("Var =/= atom");434 testObject.log(expected, actual);435 assert.deepStrictEqual(actual, expected);436}437function compare14(testObject: TestSuite.Object) {438 let expected = [new Compare(new Atom("atom"), new Atom("atom"), {exact: true, not: true})];439 let [actual, _] = ErlangJs.tokenise("atom =/= atom");440 testObject.log(expected, actual);441 assert.deepStrictEqual(actual, expected);442}443function compare15(testObject: TestSuite.Object) {444 let expected = [new Compare(null, new Atom("atom"), {exact: true, not: true})];445 let [actual, _] = ErlangJs.tokenise(" =/= atom");446 testObject.log(expected, actual);447 assert.deepStrictEqual(actual, expected);448}449function compare16(testObject: TestSuite.Object) {450 let expected = [new Compare(new Variable("Var"), null, {exact: true, not: true})];451 let [actual, _] = ErlangJs.tokenise("Var =/= ");452 testObject.log(expected, actual);453 assert.deepStrictEqual(actual, expected);454}455/*********************/456/* Boolean Operators */457/*********************/458function and1(testObject: TestSuite.Object) {459 let expected = [new BooleanOp(BooleanOp.Type.AND, new Variable("Var"), new Atom("atom"))];460 let [actual, _] = ErlangJs.tokenise("Var and atom");461 testObject.log(expected, actual);462 assert.deepStrictEqual(actual, expected);463}464function and2(testObject: TestSuite.Object) {465 let expected = [new BooleanOp(BooleanOp.Type.AND, new Atom("atom"), new Atom("atom"))];466 let [actual, _] = ErlangJs.tokenise("atom and atom");467 testObject.log(expected, actual);468 assert.deepStrictEqual(actual, expected);469}470function and3(testObject: TestSuite.Object) {471 let expected = [new BooleanOp(BooleanOp.Type.AND, null, new Atom("atom"))];472 let [actual, _] = ErlangJs.tokenise(" and atom");473 testObject.log(expected, actual);474 assert.deepStrictEqual(actual, expected);475}476function and4(testObject: TestSuite.Object) {477 let expected = [new BooleanOp(BooleanOp.Type.AND, new Variable("Var"), null)];478 let [actual, _] = ErlangJs.tokenise("Var and ");479 testObject.log(expected, actual);480 assert.deepStrictEqual(actual, expected);481}482function andAlso1(testObject: TestSuite.Object) {483 let expected = [new BooleanOp(BooleanOp.Type.ANDALSO, new Variable("Var"), new Atom("atom"))];484 let [actual, _] = ErlangJs.tokenise("Var andalso atom");485 testObject.log(expected, actual);486 assert.deepStrictEqual(actual, expected);487}488function andAlso2(testObject: TestSuite.Object) {489 let expected = [new BooleanOp(BooleanOp.Type.ANDALSO, new Atom("atom"), new Atom("atom"))];490 let [actual, _] = ErlangJs.tokenise("atom andalso atom");491 testObject.log(expected, actual);492 assert.deepStrictEqual(actual, expected);493}494function andAlso3(testObject: TestSuite.Object) {495 let expected = [new BooleanOp(BooleanOp.Type.ANDALSO, null, new Atom("atom"))];496 let [actual, _] = ErlangJs.tokenise(" andalso atom");497 testObject.log(expected, actual);498 assert.deepStrictEqual(actual, expected);499}500function andAlso4(testObject: TestSuite.Object) {501 let expected = [new BooleanOp(BooleanOp.Type.ANDALSO, new Variable("Var"), null)];502 let [actual, _] = ErlangJs.tokenise("Var andalso ");503 testObject.log(expected, actual);504 assert.deepStrictEqual(actual, expected);505}506function or1(testObject: TestSuite.Object) {507 let expected = [new BooleanOp(BooleanOp.Type.OR, new Variable("Var"), new Atom("atom"))];508 let [actual, _] = ErlangJs.tokenise("Var or atom");509 testObject.log(expected, actual);510 assert.deepStrictEqual(actual, expected);511}512function or2(testObject: TestSuite.Object) {513 let expected = [new BooleanOp(BooleanOp.Type.OR, new Atom("atom"), new Atom("atom"))];514 let [actual, _] = ErlangJs.tokenise("atom or atom");515 testObject.log(expected, actual);516 assert.deepStrictEqual(actual, expected);517}518function or3(testObject: TestSuite.Object) {519 let expected = [new BooleanOp(BooleanOp.Type.OR, null, new Atom("atom"))];520 let [actual, _] = ErlangJs.tokenise(" or atom");521 testObject.log(expected, actual);522 assert.deepStrictEqual(actual, expected);523}524function or4(testObject: TestSuite.Object) {525 let expected = [new BooleanOp(BooleanOp.Type.OR, new Variable("Var"), null)];526 let [actual, _] = ErlangJs.tokenise("Var or ");527 testObject.log(expected, actual);528 assert.deepStrictEqual(actual, expected);529}530function orElse1(testObject: TestSuite.Object) {531 let expected = [new BooleanOp(BooleanOp.Type.ORELSE, new Variable("Var"), new Atom("atom"))];532 let [actual, _] = ErlangJs.tokenise("Var orelse atom");533 testObject.log(expected, actual);534 assert.deepStrictEqual(actual, expected);535}536function orElse2(testObject: TestSuite.Object) {537 let expected = [new BooleanOp(BooleanOp.Type.ORELSE, new Atom("atom"), new Atom("atom"))];538 let [actual, _] = ErlangJs.tokenise("atom orelse atom");539 testObject.log(expected, actual);540 assert.deepStrictEqual(actual, expected);541}542function orElse3(testObject: TestSuite.Object) {543 let expected = [new BooleanOp(BooleanOp.Type.ORELSE, null, new Atom("atom"))];544 let [actual, _] = ErlangJs.tokenise(" orelse atom");545 testObject.log(expected, actual);546 assert.deepStrictEqual(actual, expected);547}548function orElse4(testObject: TestSuite.Object) {549 let expected = [new BooleanOp(BooleanOp.Type.ORELSE, new Variable("Var"), null)];550 let [actual, _] = ErlangJs.tokenise("Var orelse ");551 testObject.log(expected, actual);552 assert.deepStrictEqual(actual, expected);553}554/***************/555/* Expressions */556/***************/557function anonymousFunction1() {558 let funName = Keyword.parse(new Atom("fun"));559 let expected = [new Fun(funName, [], [new Fun.Case(new Parenthesis(), new Atom("ok"))], true)];560 let [actual, _] = ErlangJs.tokenise("fun() -> ok end");561 assert.deepStrictEqual(actual, expected);562}563function anonymousFunction2(testObject: TestSuite.Object) {564 let funName = Keyword.parse(new Atom("fun"));565 let expected = [new Fun(funName, [], [566 new Fun.Case(new Parenthesis(new Integer(1)), new Tuple(new Atom("ok"), new Atom("case1"))),567 new Fun.Case(new Parenthesis(new Integer(2)), new Tuple(new Atom("ok"), new Atom("case2"))),568 new Fun.Case(new Parenthesis(new Integer(3)), new Tuple(new Atom("ok"), new Atom("case3"))),569 new Fun.Case(new Parenthesis(new Variable("_")), new Tuple(new Atom("error"), new Atom("badarg"))),570 ], true)];571 let [actual, _] = ErlangJs.tokenise(`fun572 (1) -> {ok, case1};573 (2) -> {ok, case2};574 (3) -> {ok, case3};575 (_) -> {error, badarg}576 end`);577 testObject.log(expected, actual);578 assert.deepStrictEqual(actual, expected);579}580function anonymousFunction3(testObject: TestSuite.Object) {581 let funName = Keyword.parse(new Atom("fun"));582 let expected = [new Fun(funName, [], [583 new Fun.Case(new Parenthesis(), ...[584 new Atom("first"), new Delimiter(","),585 new Atom("second"), new Delimiter(","),586 new Atom("third"),587 ]),588 ], true)];589 let [actual, _] = ErlangJs.tokenise(`fun() ->590 first,591 second,592 third593 end`);594 testObject.log(expected, actual);595 assert.deepStrictEqual(actual, expected);596}597function addXAndY(testObject: TestSuite.Object) {598 let expected = [new Fun(599 new Atom("add"), 600 [],601 [new Fun.Case(602 new Parenthesis(new Variable("X"), new Variable("Y")),603 ...[new Add(new Variable("X"), new Variable("Y"))]604 )]605 )];606 let [actual, _] = ErlangJs.tokenise(`607 add(X, Y) ->608 X + Y.609 `);610 testObject.log(expected, actual);611 assert.deepStrictEqual(actual, expected);...
Tokeniser_spec.mjs
Source:Tokeniser_spec.mjs
...12 return {b, e, q, s, v};13 }14 it('converts the source into atomic tokens', () => {15 const input = 'foo bar -> baz';16 const tokens = tokeniser.tokenise(input);17 expect(tokens).toEqual([18 token({s: '', v: 'foo'}),19 token({s: ' ', v: 'bar'}),20 token({s: ' ', v: '->'}),21 token({s: ' ', v: 'baz'}),22 ]);23 });24 it('splits tokens at flexible boundaries', () => {25 const input = 'foo bar->baz';26 const tokens = tokeniser.tokenise(input);27 expect(tokens).toEqual([28 token({s: '', v: 'foo'}),29 token({s: ' ', v: 'bar'}),30 token({s: '', v: '->'}),31 token({s: '', v: 'baz'}),32 ]);33 });34 it('stores character numbers', () => {35 const input = 'foo bar -> baz';36 const tokens = tokeniser.tokenise(input);37 expect(tokens).toEqual([38 token({b: {ch: 0, i: 0, ln: 0}, e: {ch: 3, i: 3, ln: 0}}),39 token({b: {ch: 4, i: 4, ln: 0}, e: {ch: 7, i: 7, ln: 0}}),40 token({b: {ch: 8, i: 8, ln: 0}, e: {ch: 10, i: 10, ln: 0}}),41 token({b: {ch: 11, i: 11, ln: 0}, e: {ch: 14, i: 14, ln: 0}}),42 ]);43 });44 it('parses newlines as tokens', () => {45 const input = 'foo bar\nbaz';46 const tokens = tokeniser.tokenise(input);47 expect(tokens).toEqual([48 token({s: '', v: 'foo'}),49 token({s: ' ', v: 'bar'}),50 token({s: '', v: '\n'}),51 token({s: '', v: 'baz'}),52 ]);53 });54 it('parses windows-style newlines as tokens', () => {55 const input = 'foo bar\r\nbaz';56 const tokens = tokeniser.tokenise(input);57 expect(tokens).toEqual([58 token({s: '', v: 'foo'}),59 token({s: ' ', v: 'bar'}),60 token({s: '\r', v: '\n'}),61 token({s: '', v: 'baz'}),62 ]);63 });64 it('parses special characters as tokens', () => {65 const input = ',:!+*...abc';66 const tokens = tokeniser.tokenise(input);67 expect(tokens).toEqual([68 token({s: '', v: ','}),69 token({s: '', v: ':'}),70 token({s: '', v: '!'}),71 token({s: '', v: '+'}),72 token({s: '', v: '*'}),73 token({s: '', v: '...'}),74 token({s: '', v: 'abc'}),75 ]);76 });77 it('accepts XML tag-like syntax', () => {78 const input = 'abc <def> >> ghi <<';79 const tokens = tokeniser.tokenise(input);80 expect(tokens).toEqual([81 token({s: '', v: 'abc'}),82 token({s: ' ', v: '<'}),83 token({s: '', v: 'def>'}),84 token({s: ' ', v: '>>'}),85 token({s: ' ', v: 'ghi'}),86 token({s: ' ', v: '<<'}),87 ]);88 });89 it('stores line numbers', () => {90 const input = 'foo bar\nbaz';91 const tokens = tokeniser.tokenise(input);92 expect(tokens).toEqual([93 token({b: {ch: 0, i: 0, ln: 0}, e: {ch: 3, i: 3, ln: 0}}),94 token({b: {ch: 4, i: 4, ln: 0}, e: {ch: 7, i: 7, ln: 0}}),95 token({b: {ch: 7, i: 7, ln: 0}, e: {ch: 0, i: 8, ln: 1}}),96 token({b: {ch: 0, i: 8, ln: 1}, e: {ch: 3, i: 11, ln: 1}}),97 ]);98 });99 it('parses quoted newlines as quoted tokens', () => {100 const input = 'foo "\n" baz';101 const tokens = tokeniser.tokenise(input);102 expect(tokens).toEqual([103 token({q: false, s: '', v: 'foo'}),104 token({q: true, s: ' ', v: '\n'}),105 token({q: false, s: ' ', v: 'baz'}),106 ]);107 });108 it('removes leading and trailing whitespace', () => {109 const input = ' foo \t bar\t\n baz';110 const tokens = tokeniser.tokenise(input);111 expect(tokens).toEqual([112 token({s: ' ', v: 'foo'}),113 token({s: ' \t ', v: 'bar'}),114 token({s: '\t', v: '\n'}),115 token({s: ' ', v: 'baz'}),116 ]);117 });118 it('parses quoted strings as single tokens', () => {119 const input = 'foo "zig zag" "abc def"';120 const tokens = tokeniser.tokenise(input);121 expect(tokens).toEqual([122 token({q: false, s: '', v: 'foo'}),123 token({q: true, s: ' ', v: 'zig zag'}),124 token({q: true, s: ' ', v: 'abc def'}),125 ]);126 });127 it('does not consider single quotes as quotes', () => {128 const input = 'foo \'zig zag\'';129 const tokens = tokeniser.tokenise(input);130 expect(tokens).toEqual([131 token({q: false, s: '', v: 'foo'}),132 token({q: false, s: ' ', v: '\'zig'}),133 token({q: false, s: ' ', v: 'zag\''}),134 ]);135 });136 it('stores character positions around quoted strings', () => {137 const input = '"foo bar"';138 const tokens = tokeniser.tokenise(input);139 expect(tokens).toEqual([140 token({b: {ch: 0, i: 0, ln: 0}, e: {ch: 9, i: 9, ln: 0}}),141 ]);142 });143 it('ignores comments', () => {144 const input = 'foo # bar baz\nzig';145 const tokens = tokeniser.tokenise(input);146 expect(tokens).toEqual([147 token({s: '', v: 'foo'}),148 token({s: '', v: '\n'}),149 token({s: '', v: 'zig'}),150 ]);151 });152 it('ignores quotes within comments', () => {153 const input = 'foo # bar "baz\nzig';154 const tokens = tokeniser.tokenise(input);155 expect(tokens).toEqual([156 token({s: '', v: 'foo'}),157 token({s: '', v: '\n'}),158 token({s: '', v: 'zig'}),159 ]);160 });161 it('interprets special characters within quoted strings', () => {162 const input = 'foo "zig\\" \\\\zag\\n"';163 const tokens = tokeniser.tokenise(input);164 expect(tokens).toEqual([165 token({q: false, s: '', v: 'foo'}),166 token({q: true, s: ' ', v: 'zig" \\zag\n'}),167 ]);168 });169 it('propagates backslashes as escape characters', () => {170 const input = '"zig \\ zag"';171 const tokens = tokeniser.tokenise(input);172 expect(tokens).toEqual([173 token({q: true, s: '', v: 'zig \u001B zag'}),174 ]);175 });176 it('removes control characters everywhere', () => {177 const input = 'a\u001Bb\u0001c "a\u001Bb\u0001c"';178 const tokens = tokeniser.tokenise(input);179 expect(tokens).toEqual([180 token({q: false, s: '', v: 'abc'}),181 token({q: true, s: ' ', v: 'abc'}),182 ]);183 });184 it('maintains whitespace and newlines within quoted strings', () => {185 const input = 'foo " zig\n zag "';186 const tokens = tokeniser.tokenise(input);187 expect(tokens).toEqual([188 token({q: false, s: '', v: 'foo'}),189 token({q: true, s: ' ', v: ' zig\n zag '}),190 ]);191 });192 it('calculates line numbers consistently within quotes', () => {193 const input = 'foo\nbar "zig\nzag\na" b';194 const tokens = tokeniser.tokenise(input);195 expect(tokens).toEqual([196 token({b: {ch: 0, i: 0, ln: 0}, e: {ch: 3, i: 3, ln: 0}}),197 token({b: {ch: 3, i: 3, ln: 0}, e: {ch: 0, i: 4, ln: 1}}),198 token({b: {ch: 0, i: 4, ln: 1}, e: {ch: 3, i: 7, ln: 1}}),199 token({b: {ch: 4, i: 8, ln: 1}, e: {ch: 2, i: 19, ln: 3}}),200 token({b: {ch: 3, i: 20, ln: 3}, e: {ch: 4, i: 21, ln: 3}}),201 ]);202 });203 it('rejects unterminated quoted values', () => {204 expect(() => tokeniser.tokenise('"nope')).toThrow(new Error(205 'Unterminated literal (began at line 1, character 0)'206 ));207 });208 it('stops tokenising arrows once they become invalid', () => {209 expect(tokeniser.tokenise('foo -> bar')).toEqual([210 token({s: '', v: 'foo'}),211 token({s: ' ', v: '->'}),212 token({s: ' ', v: 'bar'}),213 ]);214 expect(tokeniser.tokenise('foo->bar')).toEqual([215 token({s: '', v: 'foo'}),216 token({s: '', v: '->'}),217 token({s: '', v: 'bar'}),218 ]);219 expect(tokeniser.tokenise('foo-xbar')).toEqual([220 token({s: '', v: 'foo'}),221 token({s: '', v: '-x'}),222 token({s: '', v: 'bar'}),223 ]);224 expect(tokeniser.tokenise('foo-xxyz')).toEqual([225 token({s: '', v: 'foo'}),226 token({s: '', v: '-x'}),227 token({s: '', v: 'xyz'}),228 ]);229 expect(tokeniser.tokenise('foo->xyz')).toEqual([230 token({s: '', v: 'foo'}),231 token({s: '', v: '->'}),232 token({s: '', v: 'xyz'}),233 ]);234 });235 });236 describe('.splitLines', () => {237 it('combines tokens', () => {238 const lines = tokeniser.splitLines([239 {q: false, s: '', v: 'abc'},240 {q: false, s: '', v: 'd'},241 ]);242 expect(lines).toEqual([243 [{q: false, s: '', v: 'abc'}, {q: false, s: '', v: 'd'}],...
simple-syntax-tokenizing.js
Source:simple-syntax-tokenizing.js
1// EXAMPLES:2// tokenise(String) => [ String | [ String | .. ] ] // arrays may be nested and/or empty3// tokenise("A + B * C") === [ "A", "+", "B", "*", "C" ]4// tokenise("function a(arg, arg)") === [ "function", "a", [ "arg", ",", "arg" ] ]5const {describe, it, Test} = require('../test-framework')6// SUBMISSION START ////////////////////////////////////////////7function tokenise(string) {8 if(string.split('(').length !== string.split(')').length) {return null}9 if (string.length = 0) {return []}10 11 const tokens = []12 let matchData = '', i = 013 14 while (i < string.length) {15 // Extract paranthesized subgroups16 if(string[i] === '(') {17 // Find index of corresponding closing parenthesis18 let j = i, nestCount = 119 while (nestCount !== 0) {20 j++21 if (string[j] === '(') {nestCount++}22 if (string[j] === ')') {nestCount--}23 }24 tokens.push(tokenise(string.substring(i+1, j)))25 i += j-i+126 } else if (string[i] === ')') {return null}27 28 // Skip over whitespace29 while(' \n\r\t'.includes(string[i]) && i < string.length) {i++}30 31 matchData = string.substr(i).match(/^(\w+|[!#$%&*+-/<=>@^_.,;]+)/)32 if (matchData) {33 matchData[0].length > 0 && tokens.push(matchData[0])34 i += matchData[0].length35 }36 }37 38 return tokens39}40// SUBMISSION END ////////////////////////////////////////////41console.log(tokenise("(((())))))(())"))42console.log(tokenise("(((()))((()))))((()()()))((())(()()))"))43console.log(tokenise("(()(()))"))44console.log(tokenise("()()"))45console.log(tokenise(""))46console.log(tokenise("()"))47console.log(tokenise(")("))48console.log(tokenise("(()(())()(()"))49console.log(tokenise("(()))()"))50console.log(tokenise("fgh^(das+_^ygrE(ssd))//UJ"))51console.log(tokenise("das+_^ygrE(ssd)"))52console.log(tokenise("ssd"))53console.log(tokenise("ds^!(sd+%%(ZZaddu))^&(())-(<=>)"))54console.log(tokenise("sd+%%(ZZaddu))^&(())-(<=>"))55console.log(tokenise("ZZaddu))^&(()"))56console.log(tokenise(")-("))57describe("Simple Syntax Tokenizing", ()=>{58 it("Example Tests", ()=>{59 Test.assertDeepEquals( tokenise(""), [] );60 Test.assertDeepEquals( tokenise("()"), [ [] ] );61 Test.assertDeepEquals( tokenise("a (Word)"), [ "a", [ "Word" ] ] );62 Test.assertDeepEquals( tokenise("These are tokens."), [ "These", "are", "tokens", "." ] );63 Test.assertDeepEquals( tokenise("add(a, b) = a + b"), [ "add", [ "a", ",", "b" ], "=", "a", "+", "b" ] );64 Test.assertDeepEquals( tokenise("A *(B ^& C)"), [ "A", "*", [ "B", "^&", "C" ] ] );65 Test.assertDeepEquals( tokenise("do $ readLine >>= putStrLn"), [ "do", "$", "readLine", ">>=", "putStrLn" ] );66 Test.assertDeepEquals( tokenise("A single mismatched bracket ("), null );67 })...
Using AI Code Generation
1var wptokenizer = require('wptokenizer');2var token = new wptokenizer();3var str = 'This is a test string to be tokenised';4var tokens = token.tokenize(str);5console.log(tokens);6var wptokenizer = require('wptokenizer');7var token = new wptokenizer();8var str = 'This is a test string to be tokenised';9var tokens = token.tokenize(str);10console.log(tokens);11var wptokenizer = require('wptokenizer');12var token = new wptokenizer();13var str = 'This is a test string to be tokenised';14var tokens = token.tokenize(str);15console.log(tokens);
Using AI Code Generation
1var wpTokenizer = require('wptokenizer');2var tokenizer = new wpTokenizer();3tokenizer.tokenise("This is a test sentence.", function(err, tokens) {4 console.log(tokens);5});6### tokenise(text, callback)
Automation Testing Tutorials
Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.
LambdaTest Learning Hubs:
- JUnit Tutorial
- TestNG Tutorial
- Webdriver Tutorial
- WebDriverIO Tutorial
- Protractor Tutorial
- Selenium 4 Tutorial
- Jenkins Tutorial
- NUnit Tutorial
- Jest Tutorial
- Playwright Tutorial
- Cypress Tutorial
- PyTest Tutorial
YouTube
You could also refer to video tutorials over LambdaTest YouTube channel to get step by step demonstration from industry experts.
Run wpt automation tests on LambdaTest cloud grid
Perform automation testing on 3000+ real desktop and mobile devices online.
Try LambdaTest Now !!
Get 100 minutes of automation test minutes FREE!!
Was this article helpful?
