aboutsummaryrefslogtreecommitdiff
path: root/driver/lexer_test.go
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--driver/lexer_test.go20
1 files changed, 18 insertions, 2 deletions
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index e0b82b1..9e5148c 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -72,9 +72,9 @@ func newEOFTokenDefault() *Token {
return newEOFToken(ModeID(spec.LexModeIDDefault.Int()), spec.LexModeNameDefault.String())
}
-func newInvalidToken(modeID ModeID, lexeme []byte) *Token {
+func newInvalidTokenDefault(lexeme []byte) *Token {
return &Token{
- ModeID: modeID,
+ ModeID: ModeID(spec.LexModeIDDefault.Int()),
ModeKindID: 0,
Lexeme: lexeme,
Invalid: true,
@@ -772,6 +772,22 @@ func TestLexer_Next(t *testing.T) {
newEOFTokenDefault(),
},
},
+ // The driver can continue lexical analysis even after it detects an invalid token.
+ {
+ lspec: &spec.LexSpec{
+ Name: "test",
+ Entries: []*spec.LexEntry{
+ newLexEntryDefaultNOP("lower", `[a-z]+`),
+ },
+ },
+ src: `foo123bar`,
+ tokens: []*Token{
+ newTokenDefault(1, 1, []byte(`foo`)),
+ newInvalidTokenDefault([]byte(`123`)),
+ newTokenDefault(1, 1, []byte(`bar`)),
+ newEOFTokenDefault(),
+ },
+ },
}
for i, tt := range test {
for compLv := compiler.CompressionLevelMin; compLv <= compiler.CompressionLevelMax; compLv++ {