TKN 1.0.0

TKN is a package to create advanced tokenizers.

Creating tokenizer

Creating tokenizer from TKN scheme

Tokenizer tokenizer = new Tokenizer(@"- token ------------
                                      MYTOKEN     | [A-Z]+
                                      SECONDTOKEN | [0-9]+
                                      THIRDTOKEN  | [A-Z_]+
                                      - group ------------------------
                                      MYGROUP | MYTOKEN (SECONDTOKEN)*");

Creating tokenizer manually

Tokenizer tokenizer = new Tokenizer();
tokenizer.Register(new TokenType("MYTOKEN", "[A-Z]+"));
tokenizer.Register(new TokenType("SECONDTOKEN", "[0-9]+"));
tokenizer.Register(new TokenGroupType("MYGROUP", new Tknex("MYTOKEN (SECONDTOKEN)*")));

Using tokenizer

string source = "HELLO 12 24 GULGDEVS TKN 12345678 THIS_IS_NOT_IN_GROUP";
TKNElementCollection elements = tokenizer.Tokenize(source);

TKN element collection

foreach (TKNElement element in elements)
{
    if (element is Token)
    {
        Token token = (Token)element;
        switch (token.Type.Name)
        {
            case "MYTOKEN":
                // Do something with token
                break;
        }
    }
    else if (element is TokenGroup)
    {
        TokenGroup group = (TokenGroup)element;
        switch (group.Type.Name)
        {
            case "MYGROUP":
                foreach (TKNElement child in group.Elements)
                {
                    // Do something with child element
                }
                break;
        }
    }
}

Console.WriteLine(elements);
/* The result is:
TKNElementCollection: {
    MYGROUP: {
        MYTOKEN: HELLO,
        SECONDTOKEN: 12,
        SECONDTOKEN: 24
    },
    MYGROUP: {
        MYTOKEN: GULGDEVS
    },
    MYGROUP: {
        MYTOKEN: TKN,
        SECONDTOKEN: 12345678
    },
    THIRDTOKEN: THIS_IS_NOT_IN_GROUP
}*/