Tokenizer.cs

Go to the documentation of this file.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace Mercury.Scanning
{
//=============================================================================
//  ITokenizer
//=============================================================================

    public interface ITokenizer
    {
        IList<Token> Tokenize(string input);
    }

//==<<  Tokenizer Options  >>==================================================

    [Flags]
    public enum TokenizerOptions
    {
        None          =  0,
        CompiledMatch =  1,
        IgnoreNumbers =  2,
        IgnoreCase    =  4,
        IgnoreQuotes  =  8,
        IgnoreSigns   = 16,
        StrictNumbers = 32, 
    }

//=============================================================================
//  Tokenizer
//=============================================================================

    public class Tokenizer : ITokenizer
    {
        //--[  Patterns  ]-----------------------------------------------------

        private static string QUOTE     = @"(?<quote>(""|(?<=(^|\s))'))(?<string>(.*?))(?<!\e)\k<quote>";
        private static string NUMBERSNG = @"(?<number>([+-]?([0-9]+(\.[0-9]+)?|\.[0-9]+)))";
        private static string NUMBER    = @"(?<number>([0-9]+(\.[0-9]+)?|\.[0-9]+))";
        private static string STRINGA   = @"(?<string>((\w";
        private static string STRINGB   = @")+";
        private static string STRINGC   = @"))";
        private static string SYMBOLA   = @"(?<symbol>([\\]?[^a-zA-Z0-9\s";
        private static string SYMBOLB   = @"]))";

        private static Tuple<string, TokenType>[] GROUPS = new[] 
        {
            Tuple.Create("number",  TokenType.Number),
            Tuple.Create("symbol",  TokenType.Symbol),
            Tuple.Create("string",  TokenType.String)
        };
        
        //--[  Private fields  ]-----------------------------------------------

        private Regex            regex;
        private HashSet<string>  kwhash;

        private Func<char, bool> forbidden = 
            (char x) => { return char.IsControl(x) || char.IsWhiteSpace(x); };

        //--[  Private methods  ]----------------------------------------------

        private void CreateRegex()
        {
            StringBuilder sb = new StringBuilder();

            if (!Options.HasFlag(TokenizerOptions.IgnoreQuotes)) 
                sb.Append(QUOTE).Append('|');

            if (!Options.HasFlag(TokenizerOptions.IgnoreNumbers))
            {
                var nstr = Options.HasFlag(TokenizerOptions.IgnoreSigns) ? NUMBER : NUMBERSNG;

                if (Options.HasFlag(TokenizerOptions.StrictNumbers))
                    sb.Append(@"(?:(^|\s))").Append(nstr).Append(@"(?:($|\s))");
                else
                    sb.Append(nstr);
                sb.Append("|");
            }

            sb.Append(STRINGA);
            if (Alpha.Length != 0) sb.Append("|[\\e]?[").Append(Alpha).Append("]");
            sb.Append(STRINGB);

            var kws = Keywords.Where(x => x.Any(c => !char.IsLetterOrDigit(c)))
                              .OrderByDescending(x => x.Length);

            foreach (string keyword in kws)
                sb.Append('|').Append(Regex.Escape(keyword));

            sb.Append(STRINGC).Append('|').Append(SYMBOLA).Append(Alpha).Append(SYMBOLB);

            RegexOptions regopt = RegexOptions.None;
            if (Options.HasFlag(TokenizerOptions.IgnoreCase))    regopt |= RegexOptions.IgnoreCase;
            if (Options.HasFlag(TokenizerOptions.CompiledMatch)) regopt |= RegexOptions.Compiled;

            regex = new Regex(sb.ToString(), regopt);
        }

        //--[  Constructors  ]-------------------------------------------------

        public Tokenizer()
            : this(new char[0])
        { }

        public Tokenizer(IEnumerable<char> alpha)
            : this(alpha, new string[0], TokenizerOptions.None)
        { }

        public Tokenizer(IEnumerable<char> alpha, IEnumerable<string> keywords, TokenizerOptions options)
        {
            if (alpha     == null) throw new ArgumentNullException("alpha");
            if (keywords  == null) throw new ArgumentNullException("keywords");

            kwhash = new HashSet<string>(options.HasFlag(TokenizerOptions.IgnoreCase)
                ? StringComparer.OrdinalIgnoreCase
                : StringComparer.Ordinal
            );

            Alpha    = alpha.ToArray();
            Keywords = keywords.ToArray();
            Options  = options;

            if (Alpha.Any(x => forbidden(x)))
                throw new ArgumentException(string.Format("Alpha contains invalid symbols"));

            for (int ix = 0; ix < Keywords.Length; ++ix)
                kwhash.Add(Keywords[ix]);

            CreateRegex();
        }

        //--[  Properties  ]---------------------------------------------------

        public string[] Keywords { get; private set; }

        public char[] Alpha { get; private set; }

        public TokenizerOptions Options { get; private set; }

        //--[  Interface implementation  ]-------------------------------------

        #region [  ITokenizer  ]

        public IList<Token> Tokenize(string input)
        {
            List<Token> results = new List<Token>();
            Match       match   = regex.Match(input);

            while(match.Success)
            {
                bool quoted = match.Groups["quote"].Captures.Count != 0;

                for (int ix = 0; ix < GROUPS.Length; ++ix)
                {
                    Group group = match.Groups[GROUPS[ix].Item1];

                    if (group.Captures.Count == 0) continue;

                    Capture   capture = group.Captures[0];
                    TokenType toktype = GROUPS[ix].Item2;

                    if (string.IsNullOrEmpty(capture.Value))
                        continue;

                    if ((toktype == TokenType.String) && !quoted && kwhash.Contains(capture.Value))
                        toktype = TokenType.Keyword;

                    results.Add(new Token(capture.Value, toktype, match.Index, quoted));
                }

                match = match.NextMatch();
            }

            return results;
        }

        #endregion

    }
}