How to use setTokenMatched method in Cucumber-gherkin

1var DIALECTS = require('./dialects');
2var Errors = require('./errors');
3var LANGUAGE_PATTERN = /^\s*#\s*language\s*:\s*([a-zA-Z\-_]+)\s*$/;
4
5module.exports = function TokenMatcher(defaultDialectName) {
6  defaultDialectName = defaultDialectName || 'en';
7
8  var dialect;
9  var dialectName;
10  var activeDocStringSeparator;
11  var indentToRemove;
12
13  function changeDialect(newDialectName, location) {
14    var newDialect = DIALECTS[newDialectName];
15    if(!newDialect) {
16      throw Errors.NoSuchLanguageException.create(newDialectName, location);
17    }
18
19    dialectName = newDialectName;
20    dialect = newDialect;
21  }
22
23  this.reset = function () {
24    if(dialectName != defaultDialectName) changeDialect(defaultDialectName);
25    activeDocStringSeparator = null;
26    indentToRemove = 0;
27  };
28
29  this.reset();
30
31  this.match_TagLine = function match_TagLine(token) {
32    if(token.line.startsWith('@')) {
33      setTokenMatched(token, 'TagLine', null, null, null, token.line.getTags());
34      return true;
35    }
36    return false;
37  };
38
39  this.match_FeatureLine = function match_FeatureLine(token) {
40    return matchTitleLine(token, 'FeatureLine', dialect.feature);
41  };
42
43  this.match_ScenarioLine = function match_ScenarioLine(token) {
44    return matchTitleLine(token, 'ScenarioLine', dialect.scenario);
45  };
46
47  this.match_ScenarioOutlineLine = function match_ScenarioOutlineLine(token) {
48    return matchTitleLine(token, 'ScenarioOutlineLine', dialect.scenarioOutline);
49  };
50
51  this.match_BackgroundLine = function match_BackgroundLine(token) {
52    return matchTitleLine(token, 'BackgroundLine', dialect.background);
53  };
54
55  this.match_ExamplesLine = function match_ExamplesLine(token) {
56    return matchTitleLine(token, 'ExamplesLine', dialect.examples);
57  };
58
59  this.match_TableRow = function match_TableRow(token) {
60    if (token.line.startsWith('|')) {
61      // TODO: indent
62      setTokenMatched(token, 'TableRow', null, null, null, token.line.getTableCells());
63      return true;
64    }
65    return false;
66  };
67
68  this.match_Empty = function match_Empty(token) {
69    if (token.line.isEmpty) {
70      setTokenMatched(token, 'Empty', null, null, 0);
71      return true;
72    }
73    return false;
74  };
75
76  this.match_Comment = function match_Comment(token) {
77    if(token.line.startsWith('#')) {
78      var text = token.line.getLineText(0); //take the entire line, including leading space
79      setTokenMatched(token, 'Comment', text, null, 0);
80      return true;
81    }
82    return false;
83  };
84
85  this.match_Language = function match_Language(token) {
86    var match;
87    if(match = token.line.trimmedLineText.match(LANGUAGE_PATTERN)) {
88      var newDialectName = match[1];
89      setTokenMatched(token, 'Language', newDialectName);
90
91      changeDialect(newDialectName, token.location);
92      return true;
93    }
94    return false;
95  };
96
97  this.match_DocStringSeparator = function match_DocStringSeparator(token) {
98    return activeDocStringSeparator == null
99      ?
100      // open
101      _match_DocStringSeparator(token, '"""', true) ||
102      _match_DocStringSeparator(token, '```', true)
103      :
104      // close
105      _match_DocStringSeparator(token, activeDocStringSeparator, false);
106  };
107
108  function _match_DocStringSeparator(token, separator, isOpen) {
109    if (token.line.startsWith(separator)) {
110      var contentType = null;
111      if (isOpen) {
112        contentType = token.line.getRestTrimmed(separator.length);
113        activeDocStringSeparator = separator;
114        indentToRemove = token.line.indent;
115      } else {
116        activeDocStringSeparator = null;
117        indentToRemove = 0;
118      }
119
120      // TODO: Use the separator as keyword. That's needed for pretty printing.
121      setTokenMatched(token, 'DocStringSeparator', contentType);
122      return true;
123    }
124    return false;
125  }
126
127  this.match_EOF = function match_EOF(token) {
128    if(token.isEof) {
129      setTokenMatched(token, 'EOF');
130      return true;
131    }
132    return false;
133  };
134
135  this.match_StepLine = function match_StepLine(token) {
136    var keywords = []
137      .concat(dialect.given)
138      .concat(dialect.when)
139      .concat(dialect.then)
140      .concat(dialect.and)
141      .concat(dialect.but);
142    var length = keywords.length;
143    for(var i = 0, keyword; i < length; i++) {
144      var keyword = keywords[i];
145
146      if (token.line.startsWith(keyword)) {
147        var title = token.line.getRestTrimmed(keyword.length);
148        setTokenMatched(token, 'StepLine', title, keyword);
149        return true;
150      }
151    }
152    return false;
153  };
154
155  this.match_Other = function match_Other(token) {
156    var text = token.line.getLineText(indentToRemove); //take the entire line, except removing DocString indents
157    setTokenMatched(token, 'Other', unescapeDocString(text), null, 0);
158    return true;
159  };
160
161  function matchTitleLine(token, tokenType, keywords) {
162    var length = keywords.length;
163    for(var i = 0, keyword; i < length; i++) {
164      var keyword = keywords[i];
165
166      if (token.line.startsWithTitleKeyword(keyword)) {
167        var title = token.line.getRestTrimmed(keyword.length + ':'.length);
168        setTokenMatched(token, tokenType, title, keyword);
169        return true;
170      }
171    }
172    return false;
173  }
174
175  function setTokenMatched(token, matchedType, text, keyword, indent, items) {
176    token.matchedType = matchedType;
177    token.matchedText = text;
178    token.matchedKeyword = keyword;
179    token.matchedIndent = (typeof indent === 'number') ? indent : (token.line == null ? 0 : token.line.indent);
180    token.matchedItems = items || [];
181
182    token.location.column = token.matchedIndent + 1;
183    token.matchedGherkinDialect = dialectName;
184  }
185
186  function unescapeDocString(text) {
187    return activeDocStringSeparator != null ? text.replace("\\\"\\\"\\\"", "\"\"\"") : text;
188  }
189};
190

Source: token_matcher.js

var DIALECTS = require('./dialects');
var Errors = require('./errors');
var LANGUAGE_PATTERN = /^\s*#\s*language\s*:\s*([a-zA-Z\-_]+)\s*$/;

module.exports = function TokenMatcher(defaultDialectName) {
  defaultDialectName = defaultDialectName || 'en';

  var dialect;
  var dialectName;
  var activeDocStringSeparator;
  var indentToRemove;

  function changeDialect(newDialectName, location) {
    var newDialect = DIALECTS[newDialectName];
    if(!newDialect) {
      throw Errors.NoSuchLanguageException.create(newDialectName, location);
    }

    dialectName = newDialectName;
    dialect = newDialect;
  }

  this.reset = function () {
    if(dialectName != defaultDialectName) changeDialect(defaultDialectName);
    activeDocStringSeparator = null;
    indentToRemove = 0;
  };

  this.reset();

  this.match_TagLine = function match_TagLine(token) {
    if(token.line.startsWith('@')) {
      setTokenMatched(token, 'TagLine', null, null, null, token.line.getTags());
      return true;
    }
    return false;
  };

  this.match_FeatureLine = function match_FeatureLine(token) {
    return matchTitleLine(token, 'FeatureLine', dialect.feature);
  };

  this.match_ScenarioLine = function match_ScenarioLine(token) {
    return matchTitleLine(token, 'ScenarioLine', dialect.scenario);
  };

  this.match_ScenarioOutlineLine = function match_ScenarioOutlineLine(token) {
    return matchTitleLine(token, 'ScenarioOutlineLine', dialect.scenarioOutline);
  };

  this.match_BackgroundLine = function match_BackgroundLine(token) {
    return matchTitleLine(token, 'BackgroundLine', dialect.background);
  };

  this.match_ExamplesLine = function match_ExamplesLine(token) {
    return matchTitleLine(token, 'ExamplesLine', dialect.examples);
  };

  this.match_TableRow = function match_TableRow(token) {
    if (token.line.startsWith('|')) {
      // TODO: indent
      setTokenMatched(token, 'TableRow', null, null, null, token.line.getTableCells());
      return true;
    }
    return false;
  };

  this.match_Empty = function match_Empty(token) {
    if (token.line.isEmpty) {
      setTokenMatched(token, 'Empty', null, null, 0);
      return true;
    }
    return false;
  };

  this.match_Comment = function match_Comment(token) {
    if(token.line.startsWith('#')) {
      var text = token.line.getLineText(0); //take the entire line, including leading space
      setTokenMatched(token, 'Comment', text, null, 0);
      return true;
    }
    return false;
  };

  this.match_Language = function match_Language(token) {
    var match;
    if(match = token.line.trimmedLineText.match(LANGUAGE_PATTERN)) {
      var newDialectName = match[1];
      setTokenMatched(token, 'Language', newDialectName);

      changeDialect(newDialectName, token.location);
      return true;
    }
    return false;
  };

  this.match_DocStringSeparator = function match_DocStringSeparator(token) {
    return activeDocStringSeparator == null
      ?
      // open
      _match_DocStringSeparator(token, '"""', true) ||
      _match_DocStringSeparator(token, '```', true)
      :
      // close
      _match_DocStringSeparator(token, activeDocStringSeparator, false);
  };

  function _match_DocStringSeparator(token, separator, isOpen) {
    if (token.line.startsWith(separator)) {
      var contentType = null;
      if (isOpen) {
        contentType = token.line.getRestTrimmed(separator.length);
        activeDocStringSeparator = separator;
        indentToRemove = token.line.indent;
      } else {
        activeDocStringSeparator = null;
        indentToRemove = 0;
      }

      // TODO: Use the separator as keyword. That's needed for pretty printing.
      setTokenMatched(token, 'DocStringSeparator', contentType);
      return true;
    }
    return false;
  }

  this.match_EOF = function match_EOF(token) {
    if(token.isEof) {
      setTokenMatched(token, 'EOF');
      return true;
    }
    return false;
  };

  this.match_StepLine = function match_StepLine(token) {
    var keywords = []
      .concat(dialect.given)
      .concat(dialect.when)
      .concat(dialect.then)
      .concat(dialect.and)
      .concat(dialect.but);
    var length = keywords.length;
    for(var i = 0, keyword; i < length; i++) {
      var keyword = keywords[i];

      if (token.line.startsWith(keyword)) {
        var title = token.line.getRestTrimmed(keyword.length);
        setTokenMatched(token, 'StepLine', title, keyword);
        return true;
      }
    }
    return false;
  };

  this.match_Other = function match_Other(token) {
    var text = token.line.getLineText(indentToRemove); //take the entire line, except removing DocString indents
    setTokenMatched(token, 'Other', unescapeDocString(text), null, 0);
    return true;
  };

  function matchTitleLine(token, tokenType, keywords) {
    var length = keywords.length;
    for(var i = 0, keyword; i < length; i++) {
      var keyword = keywords[i];

      if (token.line.startsWithTitleKeyword(keyword)) {
        var title = token.line.getRestTrimmed(keyword.length + ':'.length);
        setTokenMatched(token, tokenType, title, keyword);
        return true;
      }
    }
    return false;
  }

  function setTokenMatched(token, matchedType, text, keyword, indent, items) {
    token.matchedType = matchedType;
    token.matchedText = text;
    token.matchedKeyword = keyword;
    token.matchedIndent = (typeof indent === 'number') ? indent : (token.line == null ? 0 : token.line.indent);
    token.matchedItems = items || [];

    token.location.column = token.matchedIndent + 1;
    token.matchedGherkinDialect = dialectName;
  }

  function unescapeDocString(text) {
    return activeDocStringSeparator != null ? text.replace("\\\"\\\"\\\"", "\"\"\"") : text;
  }
};