Mercury library 1.0 Translation of CFG grammars into an object model (Bachelor's thesis).

- Main Page
- Packages
- Classes
- Files

- File List

All Classes Namespaces Files Functions Variables Enumerations Enumerator Properties
Tokenizer.cs
Go to the documentation of this file.
1 using System;
2 using System.Collections.Generic;
3 using System.Linq;
4 using System.Text;
5 using System.Text.RegularExpressions;
6
7 namespace Mercury.Scanning
8 {
9 //=============================================================================
10 // ITokenizer
11 //=============================================================================
12
14 public interface ITokenizer
15 {
19 IList<Token> Tokenize(string input);
20 }
21
22 //==<< Tokenizer Options >>==================================================
23
25 [Flags]
26 public enum TokenizerOptions
27 {
29 None = 0,
31 CompiledMatch = 1,
33 IgnoreNumbers = 2,
35 IgnoreCase = 4,
37 IgnoreQuotes = 8,
39 IgnoreSigns = 16,
41 StrictNumbers = 32,
42 }
43
44 //=============================================================================
45 // Tokenizer
46 //=============================================================================
47
71 public class Tokenizer : ITokenizer
72 {
73 //--[ Patterns ]-----------------------------------------------------
74
75 private static string QUOTE = @"(?<quote>(""|(?<=(^|\s))'))(?<string>(.*?))(?<!\e)\k<quote>";
76 private static string NUMBERSNG = @"(?<number>([+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+)))";
77 private static string NUMBER = @"(?<number>([0-9]+(\.[0-9]+)?|\.[0-9]+))";
78 private static string STRINGA = @"(?<string>((\w";
79 private static string STRINGB = @")+";
80 private static string STRINGC = @"))";
81 private static string SYMBOLA = @"(?<symbol>([\\]?[^a-zA-Z0-9\s";
82 private static string SYMBOLB = @"]))";
83
84 private static Tuple<string, TokenType>[] GROUPS = new[]
85 {
86 Tuple.Create("number", TokenType.Number),
87 Tuple.Create("symbol", TokenType.Symbol),
88 Tuple.Create("string", TokenType.String)
89 };
90
91 //--[ Private fields ]-----------------------------------------------
92
93 private Regex regex;
94 private HashSet<string> kwhash;
95
96 private Func<char, bool> forbidden =
97 (char x) => { return char.IsControl(x) || char.IsWhiteSpace(x); };
98
99 //--[ Private methods ]----------------------------------------------
100
101 private void CreateRegex()
102 {
103 StringBuilder sb = new StringBuilder();
104
105 if (!Options.HasFlag(TokenizerOptions.IgnoreQuotes))
106 sb.Append(QUOTE).Append('|');
107
108 if (!Options.HasFlag(TokenizerOptions.IgnoreNumbers))
109 {
110 var nstr = Options.HasFlag(TokenizerOptions.IgnoreSigns) ? NUMBER : NUMBERSNG;
111
112 if (Options.HasFlag(TokenizerOptions.StrictNumbers))
113 sb.Append(@"(?:(^|\s))").Append(nstr).Append(@"(?:($|\s))");
114 else
115 sb.Append(nstr);
116 sb.Append("|");
117 }
118
119 sb.Append(STRINGA);
120 if (Alpha.Length != 0) sb.Append("|[\\e]?[").Append(Alpha).Append("]");
121 sb.Append(STRINGB);
122
123 var kws = Keywords.Where(x => x.Any(c => !char.IsLetterOrDigit(c)))
124 .OrderByDescending(x => x.Length);
125
126 foreach (string keyword in kws)
127 sb.Append('|').Append(Regex.Escape(keyword));
128
129 sb.Append(STRINGC).Append('|').Append(SYMBOLA).Append(Alpha).Append(SYMBOLB);
130
131 RegexOptions regopt = RegexOptions.None;
132 if (Options.HasFlag(TokenizerOptions.IgnoreCase)) regopt |= RegexOptions.IgnoreCase;
133 if (Options.HasFlag(TokenizerOptions.CompiledMatch)) regopt |= RegexOptions.Compiled;
134
135 regex = new Regex(sb.ToString(), regopt);
136 }
137
138 //--[ Constructors ]-------------------------------------------------
139
141 public Tokenizer()
142 : this(new char[0])
143 { }
144
147 public Tokenizer(IEnumerable<char> alpha)
148 : this(alpha, new string[0], TokenizerOptions.None)
149 { }
150
158 public Tokenizer(IEnumerable<char> alpha, IEnumerable<string> keywords, TokenizerOptions options)
159 {
160 if (alpha == null) throw new ArgumentNullException("alpha");
161 if (keywords == null) throw new ArgumentNullException("keywords");
162
163 kwhash = new HashSet<string>(options.HasFlag(TokenizerOptions.IgnoreCase)
164 ? StringComparer.OrdinalIgnoreCase
165 : StringComparer.Ordinal
166 );
167
168 Alpha = alpha.ToArray();
169 Keywords = keywords.ToArray();
170 Options = options;
171
172 if (Alpha.Any(x => forbidden(x)))
173 throw new ArgumentException(string.Format("Alpha contains invalid symbols"));
174
175 for (int ix = 0; ix < Keywords.Length; ++ix)
176 kwhash.Add(Keywords[ix]);
177
178 CreateRegex();
179 }
180
181 //--[ Properties ]---------------------------------------------------
182
184 public string[] Keywords { get; private set; }
185
187 public char[] Alpha { get; private set; }
188
190 public TokenizerOptions Options { get; private set; }
191
192 //--[ Interface implementation ]-------------------------------------
193
194 #region [ ITokenizer ]
195
196 public IList<Token> Tokenize(string input)
197 {
198 List<Token> results = new List<Token>();
199 Match match = regex.Match(input);
200
201 while(match.Success)
202 {
203 bool quoted = match.Groups["quote"].Captures.Count != 0;
204
205 for (int ix = 0; ix < GROUPS.Length; ++ix)
206 {
207 Group group = match.Groups[GROUPS[ix].Item1];
208
209 if (group.Captures.Count == 0) continue;
210
211 Capture capture = group.Captures[0];
212 TokenType toktype = GROUPS[ix].Item2;
213
214 if (string.IsNullOrEmpty(capture.Value))
215 continue;
216
217 if ((toktype == TokenType.String) && !quoted && kwhash.Contains(capture.Value))
218 toktype = TokenType.Keyword;
219
220 results.Add(new Token(capture.Value, toktype, match.Index, quoted));
221 }
222
223 match = match.NextMatch();
224 }
225
226 return results;
227 }
228
229 #endregion
230
231 }
232 }
Mercury.Scanning.IgnoreNumbers
summary>Ignores case when matching patterns.
Mercury.Scanning.Tokenizer.Tokenizer
Tokenizer()
Creates a default tokenizer
Definition: Tokenizer.cs:141
Mercury.Scanning.Token
Token, a part of an input text recognized by tokenizer
Definition: Token.cs:29
Mercury.Syntax.Scanning
Edge created by input scanning
Mercury.Scanning.IgnoreSigns
summary>With this option, numbers must be separated by spaces.
Mercury.Scanning.Tokenizer
A default implementation of ITokenizer interface using Microsoft Regular Expressions. Splits tokens in the following fashion:
Definition: Tokenizer.cs:71
Mercury.Scanning.Tokenizer.Tokenizer
Tokenizer(IEnumerable< char > alpha)
Creates a tokenizer with custom alpha characters
Definition: Tokenizer.cs:147
Mercury.Scanning.CompiledMatch
summary>Will not detect numbers, but consider them strings instead.
Mercury.Scanning.Tokenizer.Tokenize
IList< Token > Tokenize(string input)
Splits source string into tokens.
Definition: Tokenizer.cs:196
Mercury.Scanning.IgnoreQuotes
summary>Detects only positive numbers.
Mercury.Scanning.TokenType
TokenType
Defines token types. The actual conditions depend on the Mercury.Scanning.ITokenizer implementation...
Definition: Token.cs:12
Mercury.Scanning.TokenizerOptions
TokenizerOptions
Options for the tokenizer
Definition: Tokenizer.cs:26
Mercury.Scanning.ITokenizer
The tokenizer interface.
Definition: Tokenizer.cs:14
Mercury.Scanning.StrictNumbers
Mercury.Scanning.IgnoreCase
summary>Treats quote characters as symbols.
Mercury.Scanning.Tokenizer.Tokenizer
Tokenizer(IEnumerable< char > alpha, IEnumerable< string > keywords, TokenizerOptions options)
Creates a tokenizer with custom alphanumeric characters, keywords and options.
Definition: Tokenizer.cs:158
Mercury.None
No failure