diff options
Diffstat (limited to 'driver/lexer_test.go')
| -rw-r--r-- | driver/lexer_test.go | 52 |
1 files changed, 52 insertions, 0 deletions
diff --git a/driver/lexer_test.go b/driver/lexer_test.go index 1d0e887..87a381c 100644 --- a/driver/lexer_test.go +++ b/driver/lexer_test.go @@ -34,6 +34,14 @@ func newLexEntryDefaultNOP(kind string, pattern string) *spec.LexEntry { } } +func newLexEntryFragment(kind string, pattern string) *spec.LexEntry { + return &spec.LexEntry{ + Kind: spec.LexKind(kind), + Pattern: spec.LexPattern(pattern), + Fragment: true, + } +} + func newTokenDefault(id int, kind string, match byteSequence) *Token { return newToken(spec.LexModeNumDefault, spec.LexModeNameDefault, id, kind, match) } @@ -477,6 +485,50 @@ func TestLexer_Next(t *testing.T) { { lspec: &spec.LexSpec{ Entries: []*spec.LexEntry{ + newLexEntryDefaultNOP("t1", "\\f{a2c}\\f{d2f}+"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabcdef", + tokens: []*Token{ + newTokenDefault(1, "t1", newByteSequence([]byte("abcdefdef"))), + newTokenDefault(1, "t1", newByteSequence([]byte("abcdef"))), + newEOFTokenDefault(), + }, + }, + { + lspec: &spec.LexSpec{ + Entries: []*spec.LexEntry{ + newLexEntryDefaultNOP("t1", "(\\f{a2c}|\\f{d2f})+"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabc", + tokens: []*Token{ + newTokenDefault(1, "t1", newByteSequence([]byte("abcdefdefabc"))), + newEOFTokenDefault(), + }, + }, + { + lspec: &spec.LexSpec{ + Entries: []*spec.LexEntry{ + newLexEntryDefaultNOP("t1", "\\f{a2c_or_d2f}+"), + newLexEntryFragment("a2c_or_d2f", "\\f{a2c}|\\f{d2f}"), + newLexEntryFragment("a2c", "abc"), + newLexEntryFragment("d2f", "def"), + }, + }, + src: "abcdefdefabc", + tokens: []*Token{ + newTokenDefault(1, "t1", newByteSequence([]byte("abcdefdefabc"))), + newEOFTokenDefault(), + }, + }, + { + lspec: &spec.LexSpec{ + Entries: []*spec.LexEntry{ newLexEntryDefaultNOP("white_space", ` *`), newLexEntry([]string{"default"}, "string_open", `"`, "string", false), newLexEntry([]string{"string"}, "escape_sequence", `\\[n"\\]`, "", false), |