aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRyo Nihei <nihei.dev@gmail.com>2021-05-02 17:38:33 +0900
committerRyo Nihei <nihei.dev@gmail.com>2021-05-02 17:38:33 +0900
commit41cb65c67ebe00025f6a37800a4fa06cc33f7f18 (patch)
treec4314ac68297bc6296f5415717c1026f9e7696a3
parentFix parser to recognize property expressions in bracket expressions (diff)
downloadtre-41cb65c67ebe00025f6a37800a4fa06cc33f7f18.tar.gz
tre-41cb65c67ebe00025f6a37800a4fa06cc33f7f18.tar.xz
Generate an invalid token from incompleted input.
When the lexer's buffer has unaccepted data and reads the EOF, the lexer treats the buffered data as an invalid token.
-rw-r--r--driver/lexer.go5
1 files changed, 5 insertions, 0 deletions
diff --git a/driver/lexer.go b/driver/lexer.go
index 356b168..8a2afd8 100644
--- a/driver/lexer.go
+++ b/driver/lexer.go
@@ -226,6 +226,11 @@ func (l *lexer) next() (*Token, error) {
l.unread(unfixedBufLen)
return tok, nil
}
+ // When `buf` has unaccepted data and reads the EOF,
+ // the lexer treats the buffered data as an invalid token.
+ if len(buf) > 0 {
+ return newInvalidToken(newByteSequence(buf)), nil
+ }
return newEOFToken(), nil
}
buf = append(buf, v)