Generate an invalid token from incompleted input.

When the lexer's buffer has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token.

1 files changed, 5 insertions, 0 deletions

diff --git a/driver/lexer.go b/driver/lexer.go
index 356b168..8a2afd8 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -226,6 +226,11 @@ func (l *lexer) next() (*Token, error) {
 				l.unread(unfixedBufLen)
 				return tok, nil
 			}
+			// When `buf` has unaccepted data and reads the EOF,
+			// the lexer treats the buffered data as an invalid token.
+			if len(buf) > 0 {
+				return newInvalidToken(newByteSequence(buf)), nil
+			}
 			return newEOFToken(), nil
 		}
 		buf = append(buf, v)