aboutsummaryrefslogtreecommitdiff
path: root/driver/lexer_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'driver/lexer_test.go')
-rw-r--r--driver/lexer_test.go228
1 files changed, 114 insertions, 114 deletions
diff --git a/driver/lexer_test.go b/driver/lexer_test.go
index 0c9f720..e48c12f 100644
--- a/driver/lexer_test.go
+++ b/driver/lexer_test.go
@@ -24,17 +24,17 @@ func TestLexer_Next(t *testing.T) {
},
src: "abb aabb aaabb babb bbabb abbbabb",
tokens: []*Token{
- newToken(1, "t1", []byte("abb")),
- newToken(2, "t2", []byte(" ")),
- newToken(1, "t1", []byte("aabb")),
- newToken(2, "t2", []byte(" ")),
- newToken(1, "t1", []byte("aaabb")),
- newToken(2, "t2", []byte(" ")),
- newToken(1, "t1", []byte("babb")),
- newToken(2, "t2", []byte(" ")),
- newToken(1, "t1", []byte("bbabb")),
- newToken(2, "t2", []byte(" ")),
- newToken(1, "t1", []byte("abbbabb")),
+ newToken(1, "t1", newByteSequence([]byte("abb"))),
+ newToken(2, "t2", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("aabb"))),
+ newToken(2, "t2", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("aaabb"))),
+ newToken(2, "t2", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("babb"))),
+ newToken(2, "t2", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("bbabb"))),
+ newToken(2, "t2", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("abbbabb"))),
newEOFToken(),
},
},
@@ -48,21 +48,21 @@ func TestLexer_Next(t *testing.T) {
},
src: "ba baaa a aaa abcd abcdcdcd cd cdcdcd",
tokens: []*Token{
- newToken(1, "t1", []byte("ba")),
- newToken(3, "t3", []byte(" ")),
- newToken(1, "t1", []byte("baaa")),
- newToken(3, "t3", []byte(" ")),
- newToken(1, "t1", []byte("a")),
- newToken(3, "t3", []byte(" ")),
- newToken(1, "t1", []byte("aaa")),
- newToken(3, "t3", []byte(" ")),
- newToken(2, "t2", []byte("abcd")),
- newToken(3, "t3", []byte(" ")),
- newToken(2, "t2", []byte("abcdcdcd")),
- newToken(3, "t3", []byte(" ")),
- newToken(2, "t2", []byte("cd")),
- newToken(3, "t3", []byte(" ")),
- newToken(2, "t2", []byte("cdcdcd")),
+ newToken(1, "t1", newByteSequence([]byte("ba"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("baaa"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("a"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(1, "t1", newByteSequence([]byte("aaa"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(2, "t2", newByteSequence([]byte("abcd"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(2, "t2", newByteSequence([]byte("abcdcdcd"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(2, "t2", newByteSequence([]byte("cd"))),
+ newToken(3, "t3", newByteSequence([]byte(" "))),
+ newToken(2, "t2", newByteSequence([]byte("cdcdcd"))),
newEOFToken(),
},
},
@@ -91,22 +91,22 @@ func TestLexer_Next(t *testing.T) {
0xf4, 0x8f, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "t1", []byte{0x00}),
- newToken(1, "t1", []byte{0x7f}),
- newToken(1, "t1", []byte{0xc2, 0x80}),
- newToken(1, "t1", []byte{0xdf, 0xbf}),
- newToken(1, "t1", []byte{0xe1, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xec, 0xbf, 0xbf}),
- newToken(1, "t1", []byte{0xed, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xed, 0x9f, 0xbf}),
- newToken(1, "t1", []byte{0xee, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xef, 0xbf, 0xbf}),
- newToken(1, "t1", []byte{0xf0, 0x90, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
- newToken(1, "t1", []byte{0xf1, 0x80, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
- newToken(1, "t1", []byte{0xf4, 0x80, 0x80, 0x80}),
- newToken(1, "t1", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
+ newToken(1, "t1", newByteSequence([]byte{0x00})),
+ newToken(1, "t1", newByteSequence([]byte{0x7f})),
+ newToken(1, "t1", newByteSequence([]byte{0xc2, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xdf, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xe1, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xec, 0xbf, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xed, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xed, 0x9f, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xee, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xef, 0xbf, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbf})),
+ newToken(1, "t1", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x80})),
+ newToken(1, "t1", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -118,17 +118,17 @@ func TestLexer_Next(t *testing.T) {
},
src: "ab.*+?|()[]",
tokens: []*Token{
- newToken(1, "t1", []byte("a")),
- newToken(1, "t1", []byte("b")),
- newToken(1, "t1", []byte(".")),
- newToken(1, "t1", []byte("*")),
- newToken(1, "t1", []byte("+")),
- newToken(1, "t1", []byte("?")),
- newToken(1, "t1", []byte("|")),
- newToken(1, "t1", []byte("(")),
- newToken(1, "t1", []byte(")")),
- newToken(1, "t1", []byte("[")),
- newToken(1, "t1", []byte("]")),
+ newToken(1, "t1", newByteSequence([]byte("a"))),
+ newToken(1, "t1", newByteSequence([]byte("b"))),
+ newToken(1, "t1", newByteSequence([]byte("."))),
+ newToken(1, "t1", newByteSequence([]byte("*"))),
+ newToken(1, "t1", newByteSequence([]byte("+"))),
+ newToken(1, "t1", newByteSequence([]byte("?"))),
+ newToken(1, "t1", newByteSequence([]byte("|"))),
+ newToken(1, "t1", newByteSequence([]byte("("))),
+ newToken(1, "t1", newByteSequence([]byte(")"))),
+ newToken(1, "t1", newByteSequence([]byte("["))),
+ newToken(1, "t1", newByteSequence([]byte("]"))),
newEOFToken(),
},
},
@@ -151,10 +151,10 @@ func TestLexer_Next(t *testing.T) {
0x7f,
}),
tokens: []*Token{
- newToken(1, "1ByteChar", []byte{0x01}),
- newToken(1, "1ByteChar", []byte{0x02}),
- newToken(1, "1ByteChar", []byte{0x7e}),
- newToken(1, "1ByteChar", []byte{0x7f}),
+ newToken(1, "1ByteChar", newByteSequence([]byte{0x01})),
+ newToken(1, "1ByteChar", newByteSequence([]byte{0x02})),
+ newToken(1, "1ByteChar", newByteSequence([]byte{0x7e})),
+ newToken(1, "1ByteChar", newByteSequence([]byte{0x7f})),
newEOFToken(),
},
},
@@ -172,10 +172,10 @@ func TestLexer_Next(t *testing.T) {
0xdf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "2ByteChar", []byte{0xc2, 0x80}),
- newToken(1, "2ByteChar", []byte{0xc2, 0x81}),
- newToken(1, "2ByteChar", []byte{0xdf, 0xbe}),
- newToken(1, "2ByteChar", []byte{0xdf, 0xbf}),
+ newToken(1, "2ByteChar", newByteSequence([]byte{0xc2, 0x80})),
+ newToken(1, "2ByteChar", newByteSequence([]byte{0xc2, 0x81})),
+ newToken(1, "2ByteChar", newByteSequence([]byte{0xdf, 0xbe})),
+ newToken(1, "2ByteChar", newByteSequence([]byte{0xdf, 0xbf})),
newEOFToken(),
},
},
@@ -190,7 +190,7 @@ func TestLexer_Next(t *testing.T) {
0xe0, 0xa0, 0x80,
}),
tokens: []*Token{
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
newEOFToken(),
},
},
@@ -208,10 +208,10 @@ func TestLexer_Next(t *testing.T) {
0xe0, 0xa0, 0xbf,
}),
tokens: []*Token{
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0xbf}),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0xbf})),
newEOFToken(),
},
},
@@ -229,10 +229,10 @@ func TestLexer_Next(t *testing.T) {
0xe0, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbf}),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -262,22 +262,22 @@ func TestLexer_Next(t *testing.T) {
0xef, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x80}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xa0, 0x81}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xe0, 0xbf, 0xbf}),
- newToken(1, "3ByteChar", []byte{0xe1, 0x80, 0x80}),
- newToken(1, "3ByteChar", []byte{0xe1, 0x80, 0x81}),
- newToken(1, "3ByteChar", []byte{0xec, 0xbf, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xec, 0xbf, 0xbf}),
- newToken(1, "3ByteChar", []byte{0xed, 0x80, 0x80}),
- newToken(1, "3ByteChar", []byte{0xed, 0x80, 0x81}),
- newToken(1, "3ByteChar", []byte{0xed, 0x9f, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xed, 0x9f, 0xbf}),
- newToken(1, "3ByteChar", []byte{0xee, 0x80, 0x80}),
- newToken(1, "3ByteChar", []byte{0xee, 0x80, 0x81}),
- newToken(1, "3ByteChar", []byte{0xef, 0xbf, 0xbe}),
- newToken(1, "3ByteChar", []byte{0xef, 0xbf, 0xbf}),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xa0, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe0, 0xbf, 0xbf})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe1, 0x80, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xe1, 0x80, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xec, 0xbf, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xec, 0xbf, 0xbf})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x80, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x80, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x9f, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xed, 0x9f, 0xbf})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xee, 0x80, 0x80})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xee, 0x80, 0x81})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xef, 0xbf, 0xbe})),
+ newToken(1, "3ByteChar", newByteSequence([]byte{0xef, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -292,7 +292,7 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0x90, 0x80, 0x80,
}),
tokens: []*Token{
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
newEOFToken(),
},
},
@@ -310,10 +310,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0x90, 0x80, 0xbf,
}),
tokens: []*Token{
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0xbf}),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0xbf})),
newEOFToken(),
},
},
@@ -331,10 +331,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0x90, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0xbf, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0xbf, 0xbf}),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0xbf, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -352,10 +352,10 @@ func TestLexer_Next(t *testing.T) {
0xf0, 0xbf, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -381,18 +381,18 @@ func TestLexer_Next(t *testing.T) {
0xf4, 0x8f, 0xbf, 0xbf,
}),
tokens: []*Token{
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf0, 0x90, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf0, 0xbf, 0xbf, 0xbf}),
- newToken(1, "4ByteChar", []byte{0xf1, 0x80, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf1, 0x80, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf3, 0xbf, 0xbf, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf3, 0xbf, 0xbf, 0xbf}),
- newToken(1, "4ByteChar", []byte{0xf4, 0x80, 0x80, 0x80}),
- newToken(1, "4ByteChar", []byte{0xf4, 0x80, 0x80, 0x81}),
- newToken(1, "4ByteChar", []byte{0xf4, 0x8f, 0xbf, 0xbe}),
- newToken(1, "4ByteChar", []byte{0xf4, 0x8f, 0xbf, 0xbf}),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0x90, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf0, 0xbf, 0xbf, 0xbf})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf1, 0x80, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf3, 0xbf, 0xbf, 0xbf})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x80})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x80, 0x80, 0x81})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbe})),
+ newToken(1, "4ByteChar", newByteSequence([]byte{0xf4, 0x8f, 0xbf, 0xbf})),
newEOFToken(),
},
},
@@ -404,7 +404,7 @@ func TestLexer_Next(t *testing.T) {
},
src: "foo9",
tokens: []*Token{
- newToken(1, "NonNumber", []byte("foo9")),
+ newToken(1, "NonNumber", newByteSequence([]byte("foo9"))),
newEOFToken(),
},
},
@@ -451,11 +451,11 @@ func TestLexer_PeekN(t *testing.T) {
expectedTokens := []*Token{
{
ID: 1,
- Match: []byte("foo"),
+ Match: newByteSequence([]byte("foo")),
},
{
ID: 2,
- Match: []byte("bar"),
+ Match: newByteSequence([]byte("bar")),
},
{
EOF: true,
generated by cgit v1.2.3 (git 2.49.0) at 2025-07-13 13:03:33 +0000