From 41cb65c67ebe00025f6a37800a4fa06cc33f7f18 Mon Sep 17 00:00:00 2001
From: Ryo Nihei <nihei.dev@gmail.com>
Date: Sun, 2 May 2021 17:38:33 +0900
Subject: Generate an invalid token from incompleted input.

When the lexer's buffer has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token.
---
 driver/lexer.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/driver/lexer.go b/driver/lexer.go
index 356b168..8a2afd8 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -226,6 +226,11 @@ func (l *lexer) next() (*Token, error) {
 				l.unread(unfixedBufLen)
 				return tok, nil
 			}
+			// When `buf` has unaccepted data and reads the EOF,
+			// the lexer treats the buffered data as an invalid token.
+			if len(buf) > 0 {
+				return newInvalidToken(newByteSequence(buf)), nil
+			}
 			return newEOFToken(), nil
 		}
 		buf = append(buf, v)
-- 
cgit v1.2.3