How to use TokenScanner method of Gherkin.TokenScanner class

Best Gherkin-dotnet code snippet using Gherkin.TokenScanner.TokenScanner

DeveroomGherkinParser.cs

Source:DeveroomGherkinParser.cs

...58        return null;59    }60    public DeveroomGherkinDocument Parse(TextReader featureFileReader, string sourceFilePath)61    {62        var tokenScanner = (ITokenScanner) new HotfixTokenScanner(featureFileReader);63        var tokenMatcher = new TokenMatcher(DialectProvider);64        _astBuilder = new DeveroomGherkinAstBuilder(sourceFilePath, () => tokenMatcher.CurrentDialect);65        var parser = new InternalParser(_astBuilder, AstBuilder.RecordStateForLine, _monitoringService);66        var gherkinDocument = parser.Parse(tokenScanner, tokenMatcher);67        CheckSemanticErrors(gherkinDocument);68        return gherkinDocument;69    }70    public DeveroomGherkinDocument GetResult() => _astBuilder.GetResult();71    private class InternalParser : Parser<DeveroomGherkinDocument>72    {73        private readonly IMonitoringService _monitoringService;74        private readonly Action<int, int> _recordStateForLine;75        public InternalParser(IAstBuilder<DeveroomGherkinDocument> astBuilder, Action<int, int> recordStateForLine,76            IMonitoringService monitoringService)77            : base(astBuilder)78        {79            _recordStateForLine = recordStateForLine;80            _monitoringService = monitoringService;81        }82        public int NullMatchToken(int state, Token token) =>83            MatchToken(state, token, new ParserContext84            {85                Errors = new List<ParserException>(),86                TokenMatcher = new AllFalseTokenMatcher(),87                TokenQueue = new Queue<Token>(),88                TokenScanner = new NullTokenScanner()89            });90        protected override int MatchToken(int state, Token token, ParserContext context)91        {92            _recordStateForLine?.Invoke(token.Location.Line, state);93            try94            {95                return base.MatchToken(state, token, context);96            }97            catch (InvalidOperationException ex)98            {99                _monitoringService.MonitorError(ex);100                throw;101            }102        }103    }104    #region Semantic Errors105    protected virtual void CheckSemanticErrors(DeveroomGherkinDocument specFlowDocument)106    {107        var errors = new List<ParserException>();108        errors.AddRange(((DeveroomGherkinAstBuilder) _astBuilder).Errors);109        if (specFlowDocument?.Feature != null)110        {111            CheckForDuplicateScenarios(specFlowDocument.Feature, errors);112            CheckForDuplicateExamples(specFlowDocument.Feature, errors);113            CheckForMissingExamples(specFlowDocument.Feature, errors);114            CheckForRulesPreSpecFlow31(specFlowDocument.Feature, errors);115        }116        // collect117        if (errors.Count == 1)118            throw errors[0];119        if (errors.Count > 1)120            throw new CompositeParserException(errors.ToArray());121    }122    private void CheckForRulesPreSpecFlow31(Feature feature, List<ParserException> errors)123    {124        //TODO: Show error when Rule keyword is used in SpecFlow v3.0 or earlier125    }126    private void CheckForDuplicateScenarios(Feature feature, List<ParserException> errors)127    {128        // duplicate scenario name129        var duplicatedScenarios = feature.FlattenScenarioDefinitions().GroupBy(sd => sd.Name, sd => sd)130            .Where(g => g.Count() > 1).ToArray();131        errors.AddRange(132            duplicatedScenarios.Select(g =>133                new SemanticParserException(134                    $"Feature file already contains a scenario with name '{g.Key}'",135                    g.ElementAt(1).Location)));136    }137    private void CheckForDuplicateExamples(Feature feature, List<ParserException> errors)138    {139        foreach (var scenarioOutline in feature.FlattenScenarioDefinitions().OfType<ScenarioOutline>())140        {141            var duplicateExamples = scenarioOutline.Examples142                .Where(e => !string.IsNullOrWhiteSpace(e.Name))143                .Where(e => e.Tags.All(t => t.Name != "ignore"))144                .GroupBy(e => e.Name, e => e).Where(g => g.Count() > 1);145            foreach (var duplicateExample in duplicateExamples)146            {147                var message =148                    $"Scenario Outline '{scenarioOutline.Name}' already contains an example with name '{duplicateExample.Key}'";149                var semanticParserException =150                    new SemanticParserException(message, duplicateExample.ElementAt(1).Location);151                errors.Add(semanticParserException);152            }153        }154    }155    private void CheckForMissingExamples(Feature feature, List<ParserException> errors)156    {157        foreach (var scenarioOutline in feature.FlattenScenarioDefinitions().OfType<ScenarioOutline>())158            if (DoesntHavePopulatedExamples(scenarioOutline))159            {160                var message = $"Scenario Outline '{scenarioOutline.Name}' has no examples defined";161                var semanticParserException = new SemanticParserException(message, scenarioOutline.Location);162                errors.Add(semanticParserException);163            }164    }165    private static bool DoesntHavePopulatedExamples(ScenarioOutline scenarioOutline)166    {167        return !scenarioOutline.Examples.Any() ||168               scenarioOutline.Examples.Any(x => x.TableBody == null || !x.TableBody.Any());169    }170    #endregion171    #region Expected tokens172    private class NullAstBuilder : IAstBuilder<DeveroomGherkinDocument>173    {174        public void Build(Token token)175        {176        }177        public void StartRule(RuleType ruleType)178        {179        }180        public void EndRule(RuleType ruleType)181        {182        }183        public DeveroomGherkinDocument GetResult() => null;184        public void Reset()185        {186        }187    }188    private class AllFalseTokenMatcher : ITokenMatcher189    {190        public bool Match_EOF(Token token) => false;191        public bool Match_Empty(Token token) => false;192        public bool Match_Comment(Token token) => false;193        public bool Match_TagLine(Token token) => false;194        public bool Match_FeatureLine(Token token) => false;195        public bool Match_RuleLine(Token token) => false;196        public bool Match_BackgroundLine(Token token) => false;197        public bool Match_ScenarioLine(Token token) => false;198        public bool Match_ExamplesLine(Token token) => false;199        public bool Match_StepLine(Token token) => false;200        public bool Match_DocStringSeparator(Token token) => false;201        public bool Match_TableRow(Token token) => false;202        public bool Match_Language(Token token) => false;203        public bool Match_Other(Token token) => false;204        public void Reset()205        {206        }207    }208    private class NullTokenScanner : ITokenScanner209    {210        public Token Read() => new(null, new Location());211    }212    public static TokenType[] GetExpectedTokens(int state, IMonitoringService monitoringService)213    {214        var parser = new InternalParser(new NullAstBuilder(), null, monitoringService)215        {216            StopAtFirstError = true217        };218        try219        {220            parser.NullMatchToken(state, new Token(null, new Location()));221        }222        catch (UnexpectedEOFException ex)...

Program.cs

Source:Program.cs

...82        {83            Feature feature;84            using (StreamReader reader = new StreamReader(file))85            {86                var scanner = new TokenScanner(new FastSourceReader(reader));87                //var scanner = new TokenScanner(new DefaultSourceReader(reader));88                feature = (Feature)parser.Parse(scanner);89            }90#if DEBUG91                //Console.WriteLine(File.ReadAllText(file));92                //Console.WriteLine("--------------");93                var formatter = new FeatureFormatter();94                Console.WriteLine(formatter.GetFeatureText(feature));95#endif96        }97        private static void DefaultGherkinParserTest(SpecFlowLangParser parser, string file)98        {99            Feature feature;100            using (StreamReader reader = new StreamReader(file))101            {...

ParserErrorsTest.cs

Source:ParserErrorsTest.cs

...43//             jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;4445//             try46//             {47//                 parser.Parse(new TokenScanner(new StringReader(@"# a comment48// Feature: Foo49//   Scenario: Bar50//     Given x51//       ```52//       unclosed docstring")), tokenMatcher);53//                 Assert.Fail("ParserException expected");54//             }55//             catch (ParserException)56//             {57//             }58//             var parsingResult2 = parser.Parse(new TokenScanner(new StringReader(@"Feature: Foo59//   Scenario: Bar60//     Given x61//       """"""62//       closed docstring63//       """"""")), tokenMatcher);64//             var astText2 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult2, jsonSerializerSettings));6566//             string expected2 = LineEndingHelper.NormalizeLineEndings(@"{67//   ""Feature"": {68//     ""Tags"": [],69//     ""Location"": {70//       ""Line"": 1,71//       ""Column"": 172//     },
...

SuccessfulParsingTests.cs

Source:SuccessfulParsingTests.cs

...24            var jsonSerializerSettings = new JsonSerializerSettings();25            jsonSerializerSettings.Formatting = Formatting.Indented;26            jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;2728            var parsingResult1 = parser.Parse(new TokenScanner(new StringReader("Feature: Test")), tokenMatcher);29            var astText1 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult1, jsonSerializerSettings));30            var parsingResult2 = parser.Parse(new TokenScanner(new StringReader("Feature: Test2")), tokenMatcher);31            var astText2 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult2, jsonSerializerSettings));3233	    string expected1 = LineEndingHelper.NormalizeLineEndings(@"{34  ""Feature"": {35    ""Tags"": [],36    ""Location"": {37      ""Line"": 1,38      ""Column"": 139    },40    ""Language"": ""en"",41    ""Keyword"": ""Feature"",42    ""Name"": ""Test"",43    ""Children"": []44  },45  ""Comments"": []46}");47	    string expected2 = LineEndingHelper.NormalizeLineEndings(@"{48  ""Feature"": {49    ""Tags"": [],50    ""Location"": {51      ""Line"": 1,52      ""Column"": 153    },54    ""Language"": ""en"",55    ""Keyword"": ""Feature"",56    ""Name"": ""Test2"",57    ""Children"": []58  },59  ""Comments"": []60}");61            Assert.Equal(expected1, astText1);62            Assert.Equal(expected2, astText2);63        }6465        [Fact]66        public void TestChangeDefaultLanguage()67        {68            var tokenMatcher = new TokenMatcher("no");69            var parser = new Parser(new AstBuilder<GherkinDocument>());70            var jsonSerializerSettings = new JsonSerializerSettings();71            jsonSerializerSettings.Formatting = Formatting.Indented;72            jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;7374            var parsingResult = parser.Parse(new TokenScanner(new StringReader("Egenskap: i18n support")), tokenMatcher);75            var astText = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult, jsonSerializerSettings));7677	    string expected = LineEndingHelper.NormalizeLineEndings(@"{78  ""Feature"": {79    ""Tags"": [],80    ""Location"": {81      ""Line"": 1,82      ""Column"": 183    },84    ""Language"": ""no"",85    ""Keyword"": ""Egenskap"",86    ""Name"": ""i18n support"",87    ""Children"": []88  },
...

TokenScanner.cs

Source:TokenScanner.cs

...11    /// If the scanner sees a `#` language header, it will reconfigure itself dynamically to look 12    /// for  Gherkin keywords for the associated language. The keywords are defined in 13    /// gherkin-languages.json.14    /// </summary>15    public class TokenScanner : ITokenScanner16    {17        protected int lineNumber = 0;18        protected readonly TextReader reader;19        public TokenScanner(TextReader reader)20        {21            this.reader = reader;22        }23        public virtual Token Read()24        {25            var line = reader.ReadLine();26            var location = new Location(++lineNumber);27            return line == null ? new Token(null, location) : new Token(new GherkinLine(line, lineNumber), location);28        }29    }30}

ParseUtils.cs

Source:ParseUtils.cs

...14        public static GherkinDocument Parse(TextReader reader, string languageCode)15        {16            var parser = new Parser();17            var dialect = new GherkinDialectProvider(languageCode);18            var tokenScanner = new TokenScanner(reader);19            var tokenMatcher = new TokenMatcher(dialect);20            21            return parser.Parse(tokenScanner, tokenMatcher);22        }23    }24}...

Parser.Extensions.cs

Source:Parser.Extensions.cs

...12        {13        }14        public GherkinDocument Parse(TextReader reader)15        {16            return Parse(new TokenScanner(reader));17        }18        public GherkinDocument Parse(string sourceFile)19        {20            using (var reader = new StreamReader(sourceFile))21            {22                return Parse(new TokenScanner(reader));23            }24        }25    }26}...

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7{8    {9        static void Main(string[] args)10        {11            TokenScanner scanner = new TokenScanner();12            scanner.Scan("this is a test");13            foreach (string token in scanner.Tokens)14            {15                Console.WriteLine(token);16            }17            Console.ReadLine();18        }19    }20}21using System;22using System.Collections.Generic;23using System.Linq;24using System.Text;25using System.Threading.Tasks;26using Gherkin;27{28    {29        static void Main(string[] args)30        {31            TokenScanner scanner = new TokenScanner();32            scanner.Scan("this is a test");33            foreach (string token in scanner.Tokens)34            {35                Console.WriteLine(token);36            }37            Console.ReadLine();38        }39    }40}41using System;42using System.Collections.Generic;43using System.Linq;44using System.Text;45using System.Threading.Tasks;46using Gherkin;47{48    {49        static void Main(string[] args)50        {51            TokenScanner scanner = new TokenScanner();52            scanner.Scan("this is a test");53            foreach (string token in scanner.Tokens)54            {55                Console.WriteLine(token);56            }57            Console.ReadLine();58        }59    }60}61using System;62using System.Collections.Generic;63using System.Linq;64using System.Text;65using System.Threading.Tasks;66using Gherkin;67{68    {69        static void Main(string[] args)70        {71            TokenScanner scanner = new TokenScanner();72            scanner.Scan("this is a test");73            foreach (string token in scanner.Tokens)74            {75                Console.WriteLine(token);76            }77            Console.ReadLine();78        }79    }80}81using System;82using System.Collections.Generic;83using System.Linq;84using System.Text;85using System.Threading.Tasks;86using Gherkin;87{

TokenScanner

Using AI Code Generation

1using Gherkin;2using System;3using System.Collections.Generic;4using System.Linq;5using System.Text;6using System.Threading.Tasks;7{8    {9        static void Main(string[] args)10        {11            TokenScanner scanner = new TokenScanner();12            scanner.scan("Hello world");13            Console.WriteLine("Hello world");14            Console.ReadLine();15        }16    }17}18using Gherkin;19using System;20using System.Collections.Generic;21using System.Linq;22using System.Text;23using System.Threading.Tasks;24{25    {26        static void Main(string[] args)27        {28            TokenScanner scanner = new TokenScanner();29            scanner.scan("Hello world");30            Console.WriteLine("Hello world");31            Console.ReadLine();32        }33    }34}35using Gherkin;36using System;37using System.Collections.Generic;38using System.Linq;39using System.Text;40using System.Threading.Tasks;41{42    {43        static void Main(string[] args)44        {45            TokenScanner scanner = new TokenScanner();46            scanner.scan("Hello world");47            Console.WriteLine("Hello world");48            Console.ReadLine();49        }50    }51}52using Gherkin;53using System;54using System.Collections.Generic;55using System.Linq;56using System.Text;57using System.Threading.Tasks;58{59    {60        static void Main(string[] args)61        {62            TokenScanner scanner = new TokenScanner();63            scanner.scan("Hello world");64            Console.WriteLine("Hello world");65            Console.ReadLine();66        }67    }68}69using Gherkin;70using System;71using System.Collections.Generic;72using System.Linq;73using System.Text;74using System.Threading.Tasks;75{76    {77        static void Main(string[] args)78        {79            TokenScanner scanner = new TokenScanner();80            scanner.scan("Hello world");81            Console.WriteLine("Hello world");82            Console.ReadLine();83        }84    }85}

TokenScanner

Using AI Code Generation

1using Gherkin;2using System;3using System.Collections.Generic;4using System.Linq;5using System.Text;6using System.Threading.Tasks;7{8    {9        static void Main(string[] args)10        {11            TokenScanner ts = new TokenScanner();12            ts.GetTokens();13        }14    }15}16using Gherkin;17using System;18using System.Collections.Generic;19using System.Linq;20using System.Text;21using System.Threading.Tasks;22{23    {24        static void Main(string[] args)25        {26            TokenScanner ts = new TokenScanner();27            ts.GetTokens();28        }29    }30}31using Gherkin;32using System;33using System.Collections.Generic;34using System.Linq;35using System.Text;36using System.Threading.Tasks;37{38    {39        static void Main(string[] args)40        {41            TokenScanner ts = new TokenScanner();42            ts.GetTokens();43        }44    }45}46using Gherkin;47using System;48using System.Collections.Generic;49using System.Linq;50using System.Text;51using System.Threading.Tasks;52{53    {54        static void Main(string[] args)55        {56            TokenScanner ts = new TokenScanner();57            ts.GetTokens();58        }59    }60}61using Gherkin;62using System;63using System.Collections.Generic;64using System.Linq;65using System.Text;66using System.Threading.Tasks;67{68    {69        static void Main(string[] args)70        {71            TokenScanner ts = new TokenScanner();72            ts.GetTokens();

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7{8    {9        static void Main(string[] args)10        {11            string str = "Feature: This is a feature";12            TokenScanner ts = new TokenScanner();13            ts.scan(str);14            Console.WriteLine(ts.getTokenType());15            Console.ReadLine();16        }17    }18}19using System;20using System.Collections.Generic;21using System.Linq;22using System.Text;23using System.Threading.Tasks;24using Gherkin;25{26    {27        static void Main(string[] args)28        {29            string str = "Scenario: This is a scenario";30            TokenScanner ts = new TokenScanner();31            ts.scan(str);32            Console.WriteLine(ts.getTokenType());33            Console.ReadLine();34        }35    }36}37using System;38using System.Collections.Generic;39using System.Linq;40using System.Text;41using System.Threading.Tasks;42using Gherkin;43{44    {45        static void Main(string[] args)46        {47            string str = "Given I have 1 dollar in my account";48            TokenScanner ts = new TokenScanner();49            ts.scan(str);50            Console.WriteLine(ts.getTokenType());51            Console.ReadLine();52        }53    }54}55using System;56using System.Collections.Generic;57using System.Linq;58using System.Text;59using System.Threading.Tasks;60using Gherkin;61{62    {63        static void Main(string[] args)64        {65            string str = "When I go to the ATM";66            TokenScanner ts = new TokenScanner();67            ts.scan(str);68            Console.WriteLine(ts.getTokenType());69            Console.ReadLine();70        }71    }72}73using System;74using System.Collections.Generic;75using System.Linq;76using System.Text;77using System.Threading.Tasks;78using Gherkin;

TokenScanner

Using AI Code Generation

1using Gherkin;2using System;3{4    {5        static void Main(string[] args)6        {7            TokenScanner scanner = new TokenScanner("Gherkin");

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6{7    {8        public void Token()9        {10            string s = "Hello World";11            string[] words = s.Split(' ');12            foreach (string word in words)13            {14                Console.WriteLine(word);15            }16        }17    }18}19using System;20using System.Collections.Generic;21using System.Linq;22using System.Text;23using System.Threading.Tasks;24{25    {26        static void Main(string[] args)27        {28            TokenScanner t = new TokenScanner();29            t.Token();30            Console.ReadKey();31        }32    }33}

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using System.IO;7{8    {9        static void Main(string[] args)10        {11            string path = @"C:\Users\Karthik\Documents\Visual Studio 2013\Projects\Gherkin\Gherkin\";12            string fileName = "Sample.feature";13            string filePath = path + fileName;14            TokenScanner ts = new TokenScanner(filePath);15            Console.WriteLine("Tokens extracted from the feature file are:");16            foreach (var item in ts.tokens)17            {18                Console.WriteLine(item);19            }20            Console.ReadLine();21        }22    }23}24using System;25using System.Collections.Generic;26using System.Linq;27using System.Text;28using System.Threading.Tasks;29using System.IO;30{31    {32        static void Main(string[] args)33        {34            string path = @"C:\Users\Karthik\Documents\Visual Studio 2013\Projects\Gherkin\Gherkin\";35            string fileName = "Sample.feature";36            string filePath = path + fileName;37            TokenScanner ts = new TokenScanner(filePath);38            Console.WriteLine("Tokens extracted from the feature file are:");39            foreach (var item in ts.tokens)40            {41                Console.WriteLine(item);42            }43            Console.ReadLine();44        }45    }46}47using System;48using System.Collections.Generic;49using System.Linq;50using System.Text;51using System.Threading.Tasks;52using System.IO;53{54    {

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.IO;4using System.Linq;5using System.Text;6using System.Threading.Tasks;7using Gherkin;8{9    {10        static void Main(string[] args)11        {12            string featureFilePath = "C:\\Users\\sudhir.joshi\\Desktop\\test.feature";13            string featureFile = File.ReadAllText(featureFilePath);14            List<string> steps = new List<string>();15            TokenScanner scanner = new TokenScanner(featureFile);16            scanner.Reset();17            string token = scanner.Peek(1);18            while (token != null)19            {20                if (token == "Given")21                {22                    steps.Add(scanner.Peek(2));23                }24                else if (token == "When")25                {26                    steps.Add(scanner.Peek(2));27                }28                else if (token == "Then")29                {30                    steps.Add(scanner.Peek(2));31                }32                else if (token == "And")33                {34                    steps.Add(scanner.Peek(2));35                }36                scanner.Advance();37                token = scanner.Peek(1);38            }39            string stepDefinitionClass = "using System;\r40using System.Collections.Generic;\r41using System.Linq;\r42using System.Text;\r43using System.Threading.Tasks;\r44using TechTalk.SpecFlow;\r45{\r46    {\r47        [Given(@"I have entered (.*) into the calculator")]\r48        public void GivenIHaveEnteredIntoTheCalculator(int p0)\r49        {\r50            ScenarioContext.Current.Pending();\r51        }\r52        [Given(@"I have entered (.*) into the calculator")]\r53        public void GivenIHaveEnteredIntoTheCalculator(int p0)\r54        {\r55            ScenarioContext.Current.Pending();\r56        }\r57        [Given(@"I have entered (.*) into the calculator")]\r58        public void GivenIHaveEnteredIntoTheCalculator(int p0)\

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.