Print the result of the lex command in JSON format

* Print the result of the lex command in JSON format.
* Print the EOF token.

1 files changed, 114 insertions, 114 deletions

diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index 0c9f720..e48c12f 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -24,17 +24,17 @@ func TestLexer_Next(t *testing.T) {
 			},
 			src: "abb aabb aaabb babb bbabb abbbabb",
 			tokens: []*Token{
-				newToken(1, "t1", []byte("abb")),
-				newToken(2, "t2", []byte(" ")),
-				newToken(1, "t1", []byte("aabb")),
-				newToken(2, "t2", []byte(" ")),
-				newToken(1, "t1", []byte("aaabb")),
-				newToken(2, "t2", []byte(" ")),
-				newToken(1, "t1", []byte("babb")),
-				newToken(2, "t2", []byte(" ")),
-				newToken(1, "t1", []byte("bbabb")),
-				newToken(2, "t2", []byte(" ")),
-				newToken(1, "t1", []byte("abbbabb")),
+				newToken(1, "t1", newByteSequence([]byte("abb"))),
+				newToken(2, "t2", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("aabb"))),
+				newToken(2, "t2", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("aaabb"))),
+				newToken(2, "t2", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("babb"))),
+				newToken(2, "t2", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("bbabb"))),
+				newToken(2, "t2", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("abbbabb"))),
 				newEOFToken(),
 			},
 		},
@@ -48,21 +48,21 @@ func TestLexer_Next(t *testing.T) {
 			},
 			src: "ba baaa a aaa abcd abcdcdcd cd cdcdcd",
 			tokens: []*Token{
-				newToken(1, "t1", []byte("ba")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(1, "t1", []byte("baaa")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(1, "t1", []byte("a")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(1, "t1", []byte("aaa")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(2, "t2", []byte("abcd")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(2, "t2", []byte("abcdcdcd")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(2, "t2", []byte("cd")),
-				newToken(3, "t3", []byte(" ")),
-				newToken(2, "t2", []byte("cdcdcd")),
+				newToken(1, "t1", newByteSequence([]byte("ba"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("baaa"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("a"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(1, "t1", newByteSequence([]byte("aaa"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(2, "t2", newByteSequence([]byte("abcd"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(2, "t2", newByteSequence([]byte("abcdcdcd"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(2, "t2", newByteSequence([]byte("cd"))),
+				newToken(3, "t3", newByteSequence([]byte(" "))),
+				newToken(2, "t2", newByteSequence([]byte("cdcdcd"))),
 				newEOFToken(),
 			},
 		},
@@ -91,22 +91,22 @@ func TestLexer_Next(t *testing.T) {
 				0xf4, 0x8f, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "t1", []byte{0x00}),
-				newToken(1, "t1", []byte{0x7f}),
-				newToken(1, "t1", []byte{0xc2, 0x80}),
-				newToken(1, "t1", []byte{0xdf, 0xbf}),
-				newToken(1, "t1", []byte{0xe1, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xec, 0xbf, 0xbf}),
-				newToken(1, "t1", []byte{0xed, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xed, 0x9f, 0xbf}),
-				newToken(1, "t1", []byte{0xee, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xef, 0xbf, 0xbf}),
-				newToken(1, "t1", []byte{0xf0, 0x90, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
-				newToken(1, "t1", []byte{0xf1, 0x80, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
-				newToken(1, "t1", []byte{0xf4, 0x80, 0x80, 0x80}),
-				newToken(1, "t1", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
+				newToken(1, "t1", newByteSequence([]byte{0x00})),
+				newToken(1, "t1", newByteSequence([]byte{0x7f})),
+				newToken(1, "t1", newByteSequence([]byte{0xc2, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xdf, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xe1, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xec, 0xbf, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xed, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xed, 0x9f, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xee, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xef, 0xbf, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbf})),
+				newToken(1, "t1", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x80})),
+				newToken(1, "t1", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -118,17 +118,17 @@ func TestLexer_Next(t *testing.T) {
 			},
 			src: "ab.*+?|()[]",
 			tokens: []*Token{
-				newToken(1, "t1", []byte("a")),
-				newToken(1, "t1", []byte("b")),
-				newToken(1, "t1", []byte(".")),
-				newToken(1, "t1", []byte("*")),
-				newToken(1, "t1", []byte("+")),
-				newToken(1, "t1", []byte("?")),
-				newToken(1, "t1", []byte("|")),
-				newToken(1, "t1", []byte("(")),
-				newToken(1, "t1", []byte(")")),
-				newToken(1, "t1", []byte("[")),
-				newToken(1, "t1", []byte("]")),
+				newToken(1, "t1", newByteSequence([]byte("a"))),
+				newToken(1, "t1", newByteSequence([]byte("b"))),
+				newToken(1, "t1", newByteSequence([]byte("."))),
+				newToken(1, "t1", newByteSequence([]byte("*"))),
+				newToken(1, "t1", newByteSequence([]byte("+"))),
+				newToken(1, "t1", newByteSequence([]byte("?"))),
+				newToken(1, "t1", newByteSequence([]byte("|"))),
+				newToken(1, "t1", newByteSequence([]byte("("))),
+				newToken(1, "t1", newByteSequence([]byte(")"))),
+				newToken(1, "t1", newByteSequence([]byte("["))),
+				newToken(1, "t1", newByteSequence([]byte("]"))),
 				newEOFToken(),
 			},
 		},
@@ -151,10 +151,10 @@ func TestLexer_Next(t *testing.T) {
 				0x7f,
 			}),
 			tokens: []*Token{
-				newToken(1, "1ByteChar", []byte{0x01}),
-				newToken(1, "1ByteChar", []byte{0x02}),
-				newToken(1, "1ByteChar", []byte{0x7e}),
-				newToken(1, "1ByteChar", []byte{0x7f}),
+				newToken(1, "1ByteChar", newByteSequence([]byte{0x01})),
+				newToken(1, "1ByteChar", newByteSequence([]byte{0x02})),
+				newToken(1, "1ByteChar", newByteSequence([]byte{0x7e})),
+				newToken(1, "1ByteChar", newByteSequence([]byte{0x7f})),
 				newEOFToken(),
 			},
 		},
@@ -172,10 +172,10 @@ func TestLexer_Next(t *testing.T) {
 				0xdf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "2ByteChar", []byte{0xc2, 0x80}),
-				newToken(1, "2ByteChar", []byte{0xc2, 0x81}),
-				newToken(1, "2ByteChar", []byte{0xdf, 0xbe}),
-				newToken(1, "2ByteChar", []byte{0xdf, 0xbf}),
+				newToken(1, "2ByteChar", newByteSequence([]byte{0xc2, 0x80})),
+				newToken(1, "2ByteChar", newByteSequence([]byte{0xc2, 0x81})),
+				newToken(1, "2ByteChar", newByteSequence([]byte{0xdf, 0xbe})),
+				newToken(1, "2ByteChar", newByteSequence([]byte{0xdf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -190,7 +190,7 @@ func TestLexer_Next(t *testing.T) {
 				0xe0, 0xa0, 0x80,
 			}),
 			tokens: []*Token{
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
 				newEOFToken(),
 			},
 		},
@@ -208,10 +208,10 @@ func TestLexer_Next(t *testing.T) {
 				0xe0, 0xa0, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0xbf}),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -229,10 +229,10 @@ func TestLexer_Next(t *testing.T) {
 				0xe0, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbf}),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -262,22 +262,22 @@ func TestLexer_Next(t *testing.T) {
 				0xef, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbf}),
-				newToken(1, "3ByteChar", []byte{0xe1, 0x80, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xe1, 0x80, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xec, 0xbf, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xec, 0xbf, 0xbf}),
-				newToken(1, "3ByteChar", []byte{0xed, 0x80, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xed, 0x80, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xed, 0x9f, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xed, 0x9f, 0xbf}),
-				newToken(1, "3ByteChar", []byte{0xee, 0x80, 0x80}),
-				newToken(1, "3ByteChar", []byte{0xee, 0x80, 0x81}),
-				newToken(1, "3ByteChar", []byte{0xef, 0xbf, 0xbe}),
-				newToken(1, "3ByteChar", []byte{0xef, 0xbf, 0xbf}),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbf})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe1, 0x80, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xe1, 0x80, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xec, 0xbf, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xec, 0xbf, 0xbf})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x80, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x80, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x9f, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x9f, 0xbf})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xee, 0x80, 0x80})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xee, 0x80, 0x81})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xef, 0xbf, 0xbe})),
+				newToken(1, "3ByteChar", newByteSequence([]byte{0xef, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -292,7 +292,7 @@ func TestLexer_Next(t *testing.T) {
 				0xf0, 0x90, 0x80, 0x80,
 			}),
 			tokens: []*Token{
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
 				newEOFToken(),
 			},
 		},
@@ -310,10 +310,10 @@ func TestLexer_Next(t *testing.T) {
 				0xf0, 0x90, 0x80, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0xbf}),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -331,10 +331,10 @@ func TestLexer_Next(t *testing.T) {
 				0xf0, 0x90, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0xbf, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0xbf, 0xbf}),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0xbf, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -352,10 +352,10 @@ func TestLexer_Next(t *testing.T) {
 				0xf0, 0xbf, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -381,18 +381,18 @@ func TestLexer_Next(t *testing.T) {
 				0xf4, 0x8f, 0xbf, 0xbf,
 			}),
 			tokens: []*Token{
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
-				newToken(1, "4ByteChar", []byte{0xf1, 0x80, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf1, 0x80, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf3, 0xbf, 0xbf, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
-				newToken(1, "4ByteChar", []byte{0xf4, 0x80, 0x80, 0x80}),
-				newToken(1, "4ByteChar", []byte{0xf4, 0x80, 0x80, 0x81}),
-				newToken(1, "4ByteChar", []byte{0xf4, 0x8f, 0xbf, 0xbe}),
-				newToken(1, "4ByteChar", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbf})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x80})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x81})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbe})),
+				newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbf})),
 				newEOFToken(),
 			},
 		},
@@ -404,7 +404,7 @@ func TestLexer_Next(t *testing.T) {
 			},
 			src: "foo9",
 			tokens: []*Token{
-				newToken(1, "NonNumber", []byte("foo9")),
+				newToken(1, "NonNumber", newByteSequence([]byte("foo9"))),
 				newEOFToken(),
 			},
 		},
@@ -451,11 +451,11 @@ func TestLexer_PeekN(t *testing.T) {
 	expectedTokens := []*Token{
 		{
 			ID:    1,
-			Match: []byte("foo"),
+			Match: newByteSequence([]byte("foo")),
 		},
 		{
 			ID:    2,
-			Match: []byte("bar"),
+			Match: newByteSequence([]byte("bar")),
 		},
 		{
 			EOF: true,