How to use TokenScanner class of Gherkin package

Best Gherkin-dotnet code snippet using Gherkin.TokenScanner

DeveroomGherkinParser.cs

Source:DeveroomGherkinParser.cs

...58        return null;59    }60    public DeveroomGherkinDocument Parse(TextReader featureFileReader, string sourceFilePath)61    {62        var tokenScanner = (ITokenScanner) new HotfixTokenScanner(featureFileReader);63        var tokenMatcher = new TokenMatcher(DialectProvider);64        _astBuilder = new DeveroomGherkinAstBuilder(sourceFilePath, () => tokenMatcher.CurrentDialect);65        var parser = new InternalParser(_astBuilder, AstBuilder.RecordStateForLine, _monitoringService);66        var gherkinDocument = parser.Parse(tokenScanner, tokenMatcher);67        CheckSemanticErrors(gherkinDocument);68        return gherkinDocument;69    }70    public DeveroomGherkinDocument GetResult() => _astBuilder.GetResult();71    private class InternalParser : Parser<DeveroomGherkinDocument>72    {73        private readonly IMonitoringService _monitoringService;74        private readonly Action<int, int> _recordStateForLine;75        public InternalParser(IAstBuilder<DeveroomGherkinDocument> astBuilder, Action<int, int> recordStateForLine,76            IMonitoringService monitoringService)77            : base(astBuilder)78        {79            _recordStateForLine = recordStateForLine;80            _monitoringService = monitoringService;81        }82        public int NullMatchToken(int state, Token token) =>83            MatchToken(state, token, new ParserContext84            {85                Errors = new List<ParserException>(),86                TokenMatcher = new AllFalseTokenMatcher(),87                TokenQueue = new Queue<Token>(),88                TokenScanner = new NullTokenScanner()89            });90        protected override int MatchToken(int state, Token token, ParserContext context)91        {92            _recordStateForLine?.Invoke(token.Location.Line, state);93            try94            {95                return base.MatchToken(state, token, context);96            }97            catch (InvalidOperationException ex)98            {99                _monitoringService.MonitorError(ex);100                throw;101            }102        }103    }104    #region Semantic Errors105    protected virtual void CheckSemanticErrors(DeveroomGherkinDocument specFlowDocument)106    {107        var errors = new List<ParserException>();108        errors.AddRange(((DeveroomGherkinAstBuilder) _astBuilder).Errors);109        if (specFlowDocument?.Feature != null)110        {111            CheckForDuplicateScenarios(specFlowDocument.Feature, errors);112            CheckForDuplicateExamples(specFlowDocument.Feature, errors);113            CheckForMissingExamples(specFlowDocument.Feature, errors);114            CheckForRulesPreSpecFlow31(specFlowDocument.Feature, errors);115        }116        // collect117        if (errors.Count == 1)118            throw errors[0];119        if (errors.Count > 1)120            throw new CompositeParserException(errors.ToArray());121    }122    private void CheckForRulesPreSpecFlow31(Feature feature, List<ParserException> errors)123    {124        //TODO: Show error when Rule keyword is used in SpecFlow v3.0 or earlier125    }126    private void CheckForDuplicateScenarios(Feature feature, List<ParserException> errors)127    {128        // duplicate scenario name129        var duplicatedScenarios = feature.FlattenScenarioDefinitions().GroupBy(sd => sd.Name, sd => sd)130            .Where(g => g.Count() > 1).ToArray();131        errors.AddRange(132            duplicatedScenarios.Select(g =>133                new SemanticParserException(134                    $"Feature file already contains a scenario with name '{g.Key}'",135                    g.ElementAt(1).Location)));136    }137    private void CheckForDuplicateExamples(Feature feature, List<ParserException> errors)138    {139        foreach (var scenarioOutline in feature.FlattenScenarioDefinitions().OfType<ScenarioOutline>())140        {141            var duplicateExamples = scenarioOutline.Examples142                .Where(e => !string.IsNullOrWhiteSpace(e.Name))143                .Where(e => e.Tags.All(t => t.Name != "ignore"))144                .GroupBy(e => e.Name, e => e).Where(g => g.Count() > 1);145            foreach (var duplicateExample in duplicateExamples)146            {147                var message =148                    $"Scenario Outline '{scenarioOutline.Name}' already contains an example with name '{duplicateExample.Key}'";149                var semanticParserException =150                    new SemanticParserException(message, duplicateExample.ElementAt(1).Location);151                errors.Add(semanticParserException);152            }153        }154    }155    private void CheckForMissingExamples(Feature feature, List<ParserException> errors)156    {157        foreach (var scenarioOutline in feature.FlattenScenarioDefinitions().OfType<ScenarioOutline>())158            if (DoesntHavePopulatedExamples(scenarioOutline))159            {160                var message = $"Scenario Outline '{scenarioOutline.Name}' has no examples defined";161                var semanticParserException = new SemanticParserException(message, scenarioOutline.Location);162                errors.Add(semanticParserException);163            }164    }165    private static bool DoesntHavePopulatedExamples(ScenarioOutline scenarioOutline)166    {167        return !scenarioOutline.Examples.Any() ||168               scenarioOutline.Examples.Any(x => x.TableBody == null || !x.TableBody.Any());169    }170    #endregion171    #region Expected tokens172    private class NullAstBuilder : IAstBuilder<DeveroomGherkinDocument>173    {174        public void Build(Token token)175        {176        }177        public void StartRule(RuleType ruleType)178        {179        }180        public void EndRule(RuleType ruleType)181        {182        }183        public DeveroomGherkinDocument GetResult() => null;184        public void Reset()185        {186        }187    }188    private class AllFalseTokenMatcher : ITokenMatcher189    {190        public bool Match_EOF(Token token) => false;191        public bool Match_Empty(Token token) => false;192        public bool Match_Comment(Token token) => false;193        public bool Match_TagLine(Token token) => false;194        public bool Match_FeatureLine(Token token) => false;195        public bool Match_RuleLine(Token token) => false;196        public bool Match_BackgroundLine(Token token) => false;197        public bool Match_ScenarioLine(Token token) => false;198        public bool Match_ExamplesLine(Token token) => false;199        public bool Match_StepLine(Token token) => false;200        public bool Match_DocStringSeparator(Token token) => false;201        public bool Match_TableRow(Token token) => false;202        public bool Match_Language(Token token) => false;203        public bool Match_Other(Token token) => false;204        public void Reset()205        {206        }207    }208    private class NullTokenScanner : ITokenScanner209    {210        public Token Read() => new(null, new Location());211    }212    public static TokenType[] GetExpectedTokens(int state, IMonitoringService monitoringService)213    {214        var parser = new InternalParser(new NullAstBuilder(), null, monitoringService)215        {216            StopAtFirstError = true217        };218        try219        {220            parser.NullMatchToken(state, new Token(null, new Location()));221        }222        catch (UnexpectedEOFException ex)...

GherkinEditorParser.cs

Source:GherkinEditorParser.cs

...87            {88                89            }90        }91        class NullTokenScanner : ITokenScanner92        {93            public Token Read()94            {95                return new Token(null, new Location());96            }97        }98        class NullAstBuilder : IAstBuilder<object>99        {100            public void Build(Token token)101            {102            }103            public void StartRule(RuleType ruleType)104            {105            }106            public void EndRule(RuleType ruleType)107            {108            }109            public object GetResult()110            {111                return null;112            }113            public void Reset()114            {115                116            }117        }118        public static TokenType[] GetExpectedTokens(int state)119        {120            var parser = new GherkinEditorParser(new GherkinTokenTagBuilder(null))121            {122                StopAtFirstError = true123            };124            try125            {126                parser.MatchToken(state, new Token(null, new Location()), new ParserContext()127                {128                    //Builder = new NullAstBuilder(),129                    Errors = new List<ParserException>(),130                    TokenMatcher = new AllFalseTokenMatcher(),131                    TokenQueue = new Queue<Token>(),132                    TokenScanner = new NullTokenScanner()133                });134            }135            catch (UnexpectedEOFException ex)136            {137                return ex.ExpectedTokenTypes.Select(type => (TokenType)Enum.Parse(typeof(TokenType), type.TrimStart('#'))).ToArray();138            }139            return new TokenType[0];140        }141    }142}...

Program.cs

Source:Program.cs

...82        {83            Feature feature;84            using (StreamReader reader = new StreamReader(file))85            {86                var scanner = new TokenScanner(new FastSourceReader(reader));87                //var scanner = new TokenScanner(new DefaultSourceReader(reader));88                feature = (Feature)parser.Parse(scanner);89            }90#if DEBUG91                //Console.WriteLine(File.ReadAllText(file));92                //Console.WriteLine("--------------");93                var formatter = new FeatureFormatter();94                Console.WriteLine(formatter.GetFeatureText(feature));95#endif96        }97        private static void DefaultGherkinParserTest(SpecFlowLangParser parser, string file)98        {99            Feature feature;100            using (StreamReader reader = new StreamReader(file))101            {...

ParserErrorsTest.cs

Source:ParserErrorsTest.cs

...43//             jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;4445//             try46//             {47//                 parser.Parse(new TokenScanner(new StringReader(@"# a comment48// Feature: Foo49//   Scenario: Bar50//     Given x51//       ```52//       unclosed docstring")), tokenMatcher);53//                 Assert.Fail("ParserException expected");54//             }55//             catch (ParserException)56//             {57//             }58//             var parsingResult2 = parser.Parse(new TokenScanner(new StringReader(@"Feature: Foo59//   Scenario: Bar60//     Given x61//       """"""62//       closed docstring63//       """"""")), tokenMatcher);64//             var astText2 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult2, jsonSerializerSettings));6566//             string expected2 = LineEndingHelper.NormalizeLineEndings(@"{67//   ""Feature"": {68//     ""Tags"": [],69//     ""Location"": {70//       ""Line"": 1,71//       ""Column"": 172//     },
...

SuccessfulParsingTests.cs

Source:SuccessfulParsingTests.cs

...24            var jsonSerializerSettings = new JsonSerializerSettings();25            jsonSerializerSettings.Formatting = Formatting.Indented;26            jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;2728            var parsingResult1 = parser.Parse(new TokenScanner(new StringReader("Feature: Test")), tokenMatcher);29            var astText1 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult1, jsonSerializerSettings));30            var parsingResult2 = parser.Parse(new TokenScanner(new StringReader("Feature: Test2")), tokenMatcher);31            var astText2 = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult2, jsonSerializerSettings));3233	    string expected1 = LineEndingHelper.NormalizeLineEndings(@"{34  ""Feature"": {35    ""Tags"": [],36    ""Location"": {37      ""Line"": 1,38      ""Column"": 139    },40    ""Language"": ""en"",41    ""Keyword"": ""Feature"",42    ""Name"": ""Test"",43    ""Children"": []44  },45  ""Comments"": []46}");47	    string expected2 = LineEndingHelper.NormalizeLineEndings(@"{48  ""Feature"": {49    ""Tags"": [],50    ""Location"": {51      ""Line"": 1,52      ""Column"": 153    },54    ""Language"": ""en"",55    ""Keyword"": ""Feature"",56    ""Name"": ""Test2"",57    ""Children"": []58  },59  ""Comments"": []60}");61            Assert.Equal(expected1, astText1);62            Assert.Equal(expected2, astText2);63        }6465        [Fact]66        public void TestChangeDefaultLanguage()67        {68            var tokenMatcher = new TokenMatcher("no");69            var parser = new Parser(new AstBuilder<GherkinDocument>());70            var jsonSerializerSettings = new JsonSerializerSettings();71            jsonSerializerSettings.Formatting = Formatting.Indented;72            jsonSerializerSettings.NullValueHandling = NullValueHandling.Ignore;7374            var parsingResult = parser.Parse(new TokenScanner(new StringReader("Egenskap: i18n support")), tokenMatcher);75            var astText = LineEndingHelper.NormalizeLineEndings(JsonConvert.SerializeObject(parsingResult, jsonSerializerSettings));7677	    string expected = LineEndingHelper.NormalizeLineEndings(@"{78  ""Feature"": {79    ""Tags"": [],80    ""Location"": {81      ""Line"": 1,82      ""Column"": 183    },84    ""Language"": ""no"",85    ""Keyword"": ""Egenskap"",86    ""Name"": ""i18n support"",87    ""Children"": []88  },
...

TokenScanner.cs

Source:TokenScanner.cs

...11    /// If the scanner sees a `#` language header, it will reconfigure itself dynamically to look 12    /// for  Gherkin keywords for the associated language. The keywords are defined in 13    /// gherkin-languages.json.14    /// </summary>15    public class TokenScanner : ITokenScanner16    {17        protected int lineNumber = 0;18        protected readonly TextReader reader;19        public TokenScanner(TextReader reader)20        {21            this.reader = reader;22        }23        public virtual Token Read()24        {25            var line = reader.ReadLine();26            var location = new Location(++lineNumber);27            return line == null ? new Token(null, location) : new Token(new GherkinLine(line, lineNumber), location);28        }29    }30}

ParseUtils.cs

Source:ParseUtils.cs

...14        public static GherkinDocument Parse(TextReader reader, string languageCode)15        {16            var parser = new Parser();17            var dialect = new GherkinDialectProvider(languageCode);18            var tokenScanner = new TokenScanner(reader);19            var tokenMatcher = new TokenMatcher(dialect);20            21            return parser.Parse(tokenScanner, tokenMatcher);22        }23    }24}...

Parser.Extensions.cs

Source:Parser.Extensions.cs

...12        {13        }14        public GherkinDocument Parse(TextReader reader)15        {16            return Parse(new TokenScanner(reader));17        }18        public GherkinDocument Parse(string sourceFile)19        {20            using (var reader = new StreamReader(sourceFile))21            {22                return Parse(new TokenScanner(reader));23            }24        }25    }26}...

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7using Gherkin.Ast;8{9    {10        static void Main(string[] args)11        {12            string[] feature = { "Feature: Test Feature", "Scenario: Test Scenario", "Given I am testing", "When I am testing", "Then I am testing", "And I am testing", "But I am testing", "Scenario Outline: Test Scenario Outline", "Given I am testing", "When I am testing", "Then I am testing", "And I am testing", "But I am testing", "Examples:", "| test |", "| test |" };13            TokenScanner ts = new TokenScanner(feature);

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7{8    {9        static void Main(string[] args)10        {11            string path = @"C:\Users\user\Documents\Visual Studio 2015\Projects\GherkinTest\GherkinTest\Feature1.feature";12            string feature = System.IO.File.ReadAllText(path);13            TokenScanner scanner = new TokenScanner(feature);14            scanner.Scan();15            Token token;16            while ((token = scanner.GetNextToken()) != null)17            {18                Console.WriteLine(token.Line + "\t" + token.Column + "\t" + token.Text);19            }20            Console.ReadLine();21        }22    }23}

TokenScanner

Using AI Code Generation

1using System;2using Gherkin.Ast;3using Gherkin;4using System.IO;5using System.Collections.Generic;6{7    {8        static void Main(string[] args)9        {10            string path = @"C:\Users\user1\source\repos\ConsoleApp1\test.feature";11            string[] lines = File.ReadAllLines(path);12            string feature = String.Join(Environment.NewLine, lines);13            var parser = new Parser();14            var feature1 = parser.Parse(feature);15            Console.WriteLine(feature1.Name);16            Console.WriteLine(feature1.Description);17            Console.WriteLine(feature1.Keyword);18            Console.WriteLine(feature1.Location.Line);19            Console.WriteLine(feature1.Location.Column);20            Console.WriteLine(feature1.Tags[0].Name);21            Console.WriteLine(feature1.Tags[0].Location.Line);22            Console.WriteLine(feature1.Tags[0].Location.Column);23            Console.WriteLine(feature1.Children[0].GetType());24            Console.WriteLine(feature1.Children[0].Location.Line);25            Console.WriteLine(feature1.Children[0].Location.Column);26            Console.WriteLine(feature1.Children[0].Keyword);27            Console.WriteLine(feature1.Children[0].Name);28            Console.WriteLine(feature1.Children[0].Description);29            Console.WriteLine(feature1.Children[0].Tags[0].Name);30            Console.WriteLine(feature1.Children[0].Tags[0].Location.Line);31            Console.WriteLine(feature1.Children[0].Tags[0].Location.Column);32            Console.WriteLine(feature1.Children[0].Steps[0].Keyword);33            Console.WriteLine(feature1.Children[0].Steps[0].Location.Line);34            Console.WriteLine(feature1.Children[0].Steps[0].Location.Column);35            Console.WriteLine(feature1.Children[0].Steps[0].Text);36            Console.WriteLine(feature1.Children[0].Steps[0].Argument.GetType());37            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Location.Line);38            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Location.Column);39            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Rows[0].Cells[0].Value);40            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Rows[0].Cells[0].Location.Line);41            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Rows[0].Cells[0].Location.Column);42            Console.WriteLine(feature1.Children[0].Steps[0].Argument.Rows[0].Cells[

TokenScanner

Using AI Code Generation

1using System;2using Gherkin.Ast;3using Gherkin.TokenScanner;4{5    {6        static void Main(string[] args)7        {8            var scanner = new TokenScanner();9            var token = scanner.Scan("Hello World");10            foreach (var t in token)11            {12                Console.WriteLine(t);13            }14            Console.ReadKey();15        }16    }17}18using System;19using Gherkin.Ast;20using Gherkin.TokenMatcher;21{22    {23        static void Main(string[] args)24        {25            var matcher = new TokenMatcher();26            var token = matcher.Match("Hello World");27            foreach (var t in token)28            {29                Console.WriteLine(t);30            }31            Console.ReadKey();32        }33    }34}35using System;36using Gherkin.Ast;37using Gherkin.TokenMatcher;38{39    {40        static void Main(string[] args)41        {42            var matcher = new TokenMatcher();43            var token = matcher.Match("Hello World");44            foreach (var t in token)45            {46                Console.WriteLine(t);47            }48            Console.ReadKey();49        }50    }51}52using System;53using Gherkin.Ast;54using Gherkin.TokenMatcher;55{56    {57        static void Main(string[] args)58        {59            var matcher = new TokenMatcher();60            var token = matcher.Match("Hello World");61            foreach (var t in token)62            {63                Console.WriteLine(t);64            }65            Console.ReadKey();66        }67    }68}69using System;70using Gherkin.Ast;71using Gherkin.TokenMatcher;72{73    {74        static void Main(string[] args)75        {76            var matcher = new TokenMatcher();77            var token = matcher.Match("Hello World");78            foreach (var t in token)79            {80                Console.WriteLine(t);81            }82            Console.ReadKey();83        }84    }85}

TokenScanner

Using AI Code Generation

1using System;2using System.IO;3using Gherkin;4{5    {6        static void Main(string[] args)7        {8            var text = File.ReadAllText("1.feature");9            var scanner = new TokenScanner(text);10            scanner.AddDefaultSeparators();11            scanner.AddSeparator("'''");12            scanner.AddSeparator("\"\"\"");13            scanner.AddSeparator("...");14            scanner.AddSeparator("~~~");15            scanner.AddSeparator("+++");16            scanner.AddSeparator("___");17            scanner.AddSeparator("###");18            scanner.AddSeparator("===");19            while (scanner.HasNext())20            {21                var token = scanner.Next();22                Console.WriteLine(token);23            }24        }25    }26}

TokenScanner

Using AI Code Generation

1using Gherkin;2using System;3{4    {5        static void Main(string[] args)6        {7            string str = "I have 5 apples";8            TokenScanner ts = new TokenScanner(str);9            ts.IgnoreWhitespace();10            ts.IgnorePattern("I");11            ts.IgnorePattern("have");12            ts.IgnorePattern("apples");13            Console.WriteLine(ts.NextInt());14            Console.ReadLine();15        }16    }17}18NextToken() Method19public string NextToken()20using Gherkin;21using System;22{23    {24        static void Main(string[] args)25        {26            string str = "I have 5 apples";27            TokenScanner ts = new TokenScanner(str);28            ts.IgnoreWhitespace();29            ts.IgnorePattern("I");30            ts.IgnorePattern("have");31            ts.IgnorePattern("apples");32            Console.WriteLine(ts.NextToken());33            Console.ReadLine();34        }35    }36}37NextDouble() Method38public double NextDouble()39using Gherkin;40using System;41{

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7using Gherkin.Ast;8{9    {10        static void Main(string[] args)11        {12            var parser = new Parser();13            var feature = parser.Parse(@"14");15            string featureName = feature.Name;16            Console.WriteLine(featureName);17            string scenarioName = feature.Children[0].Name;18            Console.WriteLine(scenarioName);19            string stepName = feature.Children[0].Steps[0].Keyword;20            Console.WriteLine(stepName);21            string stepDescription = feature.Children[0].Steps[0].Text;22            Console.WriteLine(stepDescription);23            string stepParameters = feature.Children[0].Steps[0].Argument.ToString();24            Console.WriteLine(stepParameters);25            string stepTable = feature.Children[0].Steps[0].Argument.Table.ToString();26            Console.WriteLine(stepTable);27            string stepDocstring = feature.Children[0].Steps[0].Argument.DocString.ToString();28            Console.WriteLine(stepDocstring);29            Console.ReadLine();30        }31    }32}

TokenScanner

Using AI Code Generation

1using System;2using System.Collections.Generic;3using System.Linq;4using System.Text;5using System.Threading.Tasks;6using Gherkin;7using Gherkin.Ast;8using System.IO;9{10    {11        static void Main(string[] args)12        {13            var parser = new Parser();14            var scanner = new TokenScanner();15            string filePath = "C:\\Users\\snehal\\Desktop\\GherkinParser\\GherkinParser\\1.feature";16            string text = File.ReadAllText(filePath);17            scanner.SetSource(text);18            var gherkinDocument = parser.Parse(scanner);19            var feature = gherkinDocument.Feature;20            var children = feature.Children;21            foreach (var child in children)22            {23                if (child is Background)24                {25                    var background = child as Background;26                    Console.WriteLine("Background: " + background.Name);27                    Console.WriteLine("Steps: ");28                    foreach (var step in background.Steps)29                    {30                        Console.WriteLine(step.Keyword + step.Text);31                    }32                }33                else if (child is Scenario)34                {35                    var scenario = child as Scenario;36                    Console.WriteLine("Scenario: " + scenario.Name);37                    Console.WriteLine("Steps: ");38                    foreach (var step in scenario.Steps)39                    {40                        Console.WriteLine(step.Keyword + step.Text);41                    }42                }43                else if (child is ScenarioOutline)44                {45                    var scenarioOutline = child as ScenarioOutline;46                    Console.WriteLine("Scenario Outline: " + scenarioOutline.Name);47                    Console.WriteLine("Steps: ");48                    foreach (var step in scenarioOutline.Steps)49                    {50                        Console.WriteLine(step.Keyword + step.Text);51                    }52                }53            }54        }55    }56}

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.