Mercury library 1.0 Translation of CFG grammars into an object model (Bachelor's thesis).

- Main Page
- Packages
- Classes
- Files

- File List

All Classes Namespaces Files Functions Variables Enumerations Enumerator Properties
Scanner.cs
Go to the documentation of this file.
1 using Mercury.Scanning;
2 using System;
3 using System.Collections;
4 using System.Collections.Generic;
5 using System.Linq;
6
7 namespace Mercury.Syntax
8 {
9 //=============================================================================
10 // IScanner
11 //=============================================================================
12
20 public interface IScanner
21 {
28 IEnumerable<Rule> Scan(IEnumerable<Token> tokens);
29 }
30
31 //=============================================================================
32 // CompositeScanner
33 //=============================================================================
34
42 public class CompositeScanner : IScanner, IEnumerable<IScanner>
43 {
44 //--[ Private fields ]-----------------------------------------------
45
46 private List<IScanner> scanners;
47
48 //--[ Constructors ]-------------------------------------------------
49
53 public CompositeScanner()
54 { scanners = new List<IScanner>(); }
55
60 public CompositeScanner(IEnumerable<IScanner> scanners)
61 : this()
62 { foreach (var scanner in scanners) Add(scanner); }
63
64 //--[ Methods ]------------------------------------------------------
65
72 public IScanner Add(IScanner scanner)
73 {
74 if (scanners.Contains(scanner)) return this;
75
76 scanners.Add(scanner);
77 return this;
78 }
79
83 public IScanner Remove(IScanner scanner)
84 {
85 scanners.Remove(scanner);
86 return this;
87 }
88
89 //--[ Interface implementation ]-------------------------------------
90
91 #region[ IScanner ]
92
93 public IEnumerable<Rule> Scan(IEnumerable<Token> tokens)
94 { return scanners.Select(scanner => scanner.Scan(tokens)).Aggregate((x, y) => x.Concat(y)); }
95
96 #endregion
97
98 #region [ IEnumerable ]
99
100 public IEnumerator<IScanner> GetEnumerator() { return scanners.GetEnumerator(); }
101
102 IEnumerator IEnumerable.GetEnumerator() { return scanners.GetEnumerator(); }
103
104 #endregion
105
106 }
107
108 //=============================================================================
109 // BasicScannerMap
110 //=============================================================================
111
112 [Flags]
113 public enum Mapping
114 {
116 None = 0,
118 AsSymbol = 1,
120 AsKeyword = 2,
122 AsNumber = 4,
124 AsString = 8,
126 All = 15,
127 };
128
132 public class BasicScannerMap
133 {
134 //--[ Private fields ]-----------------------------------------------
135
136 private IDictionary<TokenType, Mapping> map;
137
138 //--[ Constructors ]-------------------------------------------------
139
145 public BasicScannerMap(IDictionary<TokenType, Mapping> map)
146 {
147 if (map == null) throw new ArgumentNullException("map");
148
149 this.map = map;
150 }
151
152 //--[ Static methods ]-----------------------------------------------
153
154 #region [ Predefined maps ]
155
157 public static BasicScannerMap Identity
158 { get { return new BasicScannerMap(new Dictionary<TokenType, Mapping> {
159 { TokenType.Symbol, Mapping.AsSymbol },
160 { TokenType.Number, Mapping.AsNumber },
161 { TokenType.Keyword, Mapping.AsKeyword },
162 { TokenType.String, Mapping.AsString },
163 }); } }
164
166 public static BasicScannerMap NumberOrString
167 { get { return new BasicScannerMap(new Dictionary<TokenType, Mapping> {
168 { TokenType.Symbol, Mapping.AsString },
169 { TokenType.Number, Mapping.AsNumber },
170 { TokenType.Keyword, Mapping.AsString },
171 { TokenType.String, Mapping.AsString },
172 }); } }
173
175 public static BasicScannerMap All
176 { get { return new BasicScannerMap(new Dictionary<TokenType, Mapping> {
177 { TokenType.Symbol, Mapping.All },
178 { TokenType.Number, Mapping.All },
179 { TokenType.Keyword, Mapping.All },
180 { TokenType.String, Mapping.All },
181 }); } }
182
188 public static BasicScannerMap SingleMap(Mapping custom)
189 { return new BasicScannerMap(new Dictionary<TokenType, Mapping> {
190 { TokenType.Symbol, custom },
191 { TokenType.Number, custom },
192 { TokenType.Keyword, custom },
193 { TokenType.String, custom },
194 }); }
195
196 #endregion
197
198 //--[ Indexer ]------------------------------------------------------
199
200 public Mapping this[TokenType ttype] { get { return GetMapping(ttype); } }
201
202 //--[ Methods ]------------------------------------------------------
203
209 public Mapping GetMapping(TokenType ttype)
210 {
211 Mapping mapping;
212 if (map.TryGetValue(ttype, out mapping))
213 return mapping;
214
215 return Mapping.None;
216 }
217
218 }
219
220 //=============================================================================
221 // BasicScanner
222 //=============================================================================
223
229 public class BasicScanner : IScanner
230 {
231 //--[ Private fields ]-----------------------------------------------
232
233 private BasicScannerMap map;
234
235 private Rule MakeRule(string lhs, Token rhs)
236 { return new Rule(new Symbol(lhs), new[] { new Symbol(rhs) }); }
237
238 //--[ Constructors ]-------------------------------------------------
239
245 public BasicScanner(BasicScannerMap map)
246 {
247 if (map == null) throw new ArgumentNullException("map");
248
249 this.map = map;
250 }
251
252 //--[ Interface implementation ]-------------------------------------
253
254 #region [ IScanner ]
255
256 public IEnumerable<Rule> Scan(IEnumerable<Token> tokens)
257 {
258 foreach(Token token in tokens)
259 {
260 Mapping mapping = map[token.Type];
261
262 if (mapping.HasFlag(Mapping.AsKeyword)) yield return MakeRule("@Keyword", token);
263 if (mapping.HasFlag(Mapping.AsNumber)) yield return MakeRule("@Number", token);
264 if (mapping.HasFlag(Mapping.AsString)) yield return MakeRule("@String", token);
265 if (mapping.HasFlag(Mapping.AsSymbol)) yield return MakeRule("@Symbol", token);
266 }
267 }
268
269 #endregion
270
271 }
272 }
Mercury.Syntax.BasicScanner
Generates four Nonterminals @Symbol, @Keyword, @Number and @String based on the given map and types o...
Definition: Scanner.cs:229
Mercury.Syntax.BasicScannerMap
Contains mapping between TokenType and Mercury.Syntax.Mapping of Mercury.Syntax.BasicScanner.
Definition: Scanner.cs:132
Mercury.Syntax.All
Maps token to symbol, keyword, number and string
Mercury.Syntax.BasicScanner.Scan
IEnumerable< Rule > Scan(IEnumerable< Token > tokens)
Scans input tokens and creates additional rules for parsing.
Definition: Scanner.cs:256
Mercury.Syntax.CompositeScanner
Mercury.Syntax.IScanner implementation that behaves like container of scanners using the Composite De...
Definition: Scanner.cs:42
Mercury.Scanning.Token
Token, a part of an input text recognized by tokenizer
Definition: Token.cs:29
Mercury.Syntax.AsNumber
Maps token to number (@Number -> token)
Mercury.Syntax.AsSymbol
Maps token to symbol (@Symbol -> token)
Mercury.Syntax.BasicScannerMap.GetMapping
Mapping GetMapping(TokenType ttype)
Returns the mapping for a given TokenType.
Definition: Scanner.cs:209
Mercury.Syntax.CompositeScanner.Add
IScanner Add(IScanner scanner)
Adds the specified scanner to the container. If the scanner is already present, this method will have...
Definition: Scanner.cs:72
Mercury.Syntax.Mapping
Mapping
Definition: Scanner.cs:113
Mercury.Syntax.CompositeScanner.Remove
IScanner Remove(IScanner scanner)
Removes the scanner from the container.
Definition: Scanner.cs:83
Mercury.Syntax.CompositeScanner.GetEnumerator
IEnumerator< IScanner > GetEnumerator()
Definition: Scanner.cs:100
Mercury.Syntax.AsString
Maps token to string (@String -> token)
Mercury.Syntax.AsKeyword
Maps token to keyword (@Keyword -> token)
Mercury.Syntax.BasicScannerMap.SingleMap
static BasicScannerMap SingleMap(Mapping custom)
Creates a mapping for all TokenTypes to custom Mapping
Definition: Scanner.cs:188
Mercury.Syntax.CompositeScanner.Scan
IEnumerable< Rule > Scan(IEnumerable< Token > tokens)
Scans input tokens and creates additional rules for parsing.
Definition: Scanner.cs:93
Mercury.Scanning.TokenType
TokenType
Defines token types. The actual conditions depend on the Mercury.Scanning.ITokenizer implementation...
Definition: Token.cs:12
Mercury.Syntax.CompositeScanner.CompositeScanner
CompositeScanner()
Creates an empty Composite scanner.
Definition: Scanner.cs:53
Mercury.Syntax.Rule
Represents a Context-Free Grammar rule. This is an immutable class.
Definition: Rule.cs:23
Mercury.Syntax.IScanner
Scanner scans input tokens and creates additional rules for parsing.
Definition: Scanner.cs:20
Mercury.Syntax.BasicScannerMap.BasicScannerMap
BasicScannerMap(IDictionary< TokenType, Mapping > map)
Creates a new instance of Mercury.Syntax.BasicScannerMap.
Definition: Scanner.cs:145
Mercury.Syntax.Symbol
Represents a single symbol of the grammar. This is an immutable class. For a symbol name to be val...
Definition: Symbol.cs:51
Mercury.Syntax.CompositeScanner.CompositeScanner
CompositeScanner(IEnumerable< IScanner > scanners)
Creates a scanner initialized by source scanners.
Definition: Scanner.cs:60
Mercury.None
No failure
Mercury.Syntax.BasicScanner.BasicScanner
BasicScanner(BasicScannerMap map)
Initializes a new instance of the BasicScanner class.
Definition: Scanner.cs:245